diff --git a/.github/conan/compatibility.py b/.github/conan/compatibility.py index cb6d69b7b8..004a29a1c4 100644 --- a/.github/conan/compatibility.py +++ b/.github/conan/compatibility.py @@ -69,10 +69,7 @@ def compatibility(conanfile): if current_version is not None: min_version = 13 max_version = 21 # support up to Xcode 27 with apple-clang 21 for now - candidate_versions = [ - str(v) for v in range(min_version, max_version + 1) - if v != current_version - ] + candidate_versions = [str(v) for v in range(min_version, max_version + 1) if v != current_version] factors.append([{"compiler.version": v} for v in candidate_versions]) combinations = _factors_combinations(factors) diff --git a/.github/github_scripts/github_actions_cache_cleanup.py b/.github/github_scripts/github_actions_cache_cleanup.py index 8e70e95383..3ba4496d0e 100755 --- a/.github/github_scripts/github_actions_cache_cleanup.py +++ b/.github/github_scripts/github_actions_cache_cleanup.py @@ -1,11 +1,12 @@ #!/bin/python3 -import requests -import re import argparse import logging -from typing import Dict, List +import re + +import requests +logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) @@ -13,9 +14,9 @@ class GithubRequestSender: def __init__(self, token: str, repository: str): self.headers = { - 'Accept': 'application/vnd.github+json', - 'Authorization': 'Bearer ' + token, - 'X-GitHub-Api-Version': '2022-11-28' + "Accept": "application/vnd.github+json", + "Authorization": "Bearer " + token, + "X-GitHub-Api-Version": "2022-11-28", } self.repository = repository @@ -25,7 +26,7 @@ def _send_get_request(self, url: str): return response.json() - def _send_delete_request(self, url: str, params: Dict[str, str]): + def _send_delete_request(self, url: str, params: dict[str, str]): response = requests.delete(url, headers=self.headers, params=params) response.raise_for_status() @@ -58,13 +59,13 @@ class GithubActionsCacheCleaner: def __init__(self, token: str, repository: str): self.github_request_sender = GithubRequestSender(token, repository) - def _list_open_pr_ids(self) -> List[str]: + def _list_open_pr_ids(self) -> list[str]: open_tickets = [] for request in self.github_request_sender.list_open_pull_requests(): - open_tickets.append(request['number']) + open_tickets.append(request["number"]) return open_tickets - def _get_cache_entries(self) -> List[CacheEntry]: + def _get_cache_entries(self) -> list[CacheEntry]: entries = [] json_result = self.github_request_sender.list_caches() for json_entry in json_result["actions_caches"]: @@ -72,10 +73,15 @@ def _get_cache_entries(self) -> List[CacheEntry]: return entries def _is_pr_already_closed(self, entry, open_tickets): - match = re.search(r'refs/pull/([\d]+)/merge', entry.key) + match = re.search(r"refs/pull/([\d]+)/merge", entry.key) return match and int(match[1]) not in open_tickets - def _remove_non_latest_branch_caches(self, entry: CacheEntry, latest_branch_cache_map: Dict[str, CacheEntry], removable_entries: List[str]): + def _remove_non_latest_branch_caches( + self, + entry: CacheEntry, + latest_branch_cache_map: dict[str, CacheEntry], + removable_entries: list[str], + ): cache_mapping_key = "-".join(entry.key.split("-")[0:-1]) if cache_mapping_key not in latest_branch_cache_map: latest_branch_cache_map[cache_mapping_key] = entry @@ -86,9 +92,9 @@ def _remove_non_latest_branch_caches(self, entry: CacheEntry, latest_branch_cach else: removable_entries.append(entry.key) - def _get_removable_cache_entries(self) -> List[str]: + def _get_removable_cache_entries(self) -> list[str]: removable_entries = [] - latest_branch_cache_map = dict() + latest_branch_cache_map = {} open_prs = self._list_open_pr_ids() for entry in self._get_cache_entries(): if self._is_pr_already_closed(entry, open_prs): @@ -99,9 +105,9 @@ def _get_removable_cache_entries(self) -> List[str]: return removable_entries - def _remove_cache_entries(self, entries_to_remove: List[str]): + def _remove_cache_entries(self, entries_to_remove: list[str]): for key in entries_to_remove: - logging.info('Removing cache entry: %s', key) + logger.info("Removing cache entry: %s", key) self.github_request_sender.delete_cache(key) def remove_obsolete_cache_entries(self): @@ -109,9 +115,17 @@ def remove_obsolete_cache_entries(self): if __name__ == "__main__": - parser = argparse.ArgumentParser(prog='Github Actions Cache Cleaner', description='Cleans up obsolete cache entries of github actions') - parser.add_argument('-r', '--repository', help='The name of the repository in / format', required=True) - parser.add_argument('-t', '--token', help='Github token for API access', required=True) + parser = argparse.ArgumentParser( + prog="Github Actions Cache Cleaner", + description="Cleans up obsolete cache entries of github actions", + ) + parser.add_argument( + "-r", + "--repository", + help="The name of the repository in / format", + required=True, + ) + parser.add_argument("-t", "--token", help="Github token for API access", required=True) args = parser.parse_args() cache_cleaner = GithubActionsCacheCleaner(args.token, args.repository) cache_cleaner.remove_obsolete_cache_entries() diff --git a/.github/github_scripts/github_actions_cache_cleanup_tests.py b/.github/github_scripts/github_actions_cache_cleanup_tests.py index 2f2f7772e7..716a7ebcb3 100755 --- a/.github/github_scripts/github_actions_cache_cleanup_tests.py +++ b/.github/github_scripts/github_actions_cache_cleanup_tests.py @@ -2,6 +2,7 @@ import unittest from unittest.mock import MagicMock + from github_actions_cache_cleanup import GithubActionsCacheCleaner @@ -21,7 +22,7 @@ def create_mock_github_request_sender(self): { "number": 229, "title": "MINIFICPP-123 TEST3", - } + }, ] mock.list_open_pull_requests.return_value = open_pull_requests caches = { @@ -57,7 +58,7 @@ def create_mock_github_request_sender(self): { "id": 55555, "key": "ubuntu-24.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567", - } + }, ] } mock.list_caches = MagicMock() @@ -102,7 +103,7 @@ def create_empty_open_pr_mock_github_request_sender(self): { "id": 55555, "key": "ubuntu-24.04-all-clang-ccache-refs/heads/main-2f4d283f5bc894af8dfc295e5976a5f1b4664567", - } + }, ] } mock.list_caches = MagicMock() @@ -125,13 +126,11 @@ def create_empty_caches_mock_github_request_sender(self): { "number": 229, "title": "MINIFICPP-123 TEST3", - } + }, ] mock.list_open_pull_requests.return_value = open_pull_requests mock.list_caches = MagicMock() - caches = { - "actions_caches": [] - } + caches = {"actions_caches": []} mock.list_caches.return_value = caches mock.delete_cache = MagicMock() return mock @@ -140,30 +139,41 @@ def test_cache_cleanup(self): cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp") cleaner.github_request_sender = self.create_mock_github_request_sender() cleaner.remove_obsolete_cache_entries() - self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]), - {"macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123", - "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456", - "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", - "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"}) + self.assertEqual( + {call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list}, + { + "macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123", + "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456", + "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", + "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456", + }, + ) def test_cache_cleanup_with_zero_open_prs(self): cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp") cleaner.github_request_sender = self.create_empty_open_pr_mock_github_request_sender() cleaner.remove_obsolete_cache_entries() - self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]), - {"macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123", - "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", - "ubuntu-24.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487", - "macos-xcode-ccache-refs/pull/227/merge-2d6d283f5bc894af8dfc295e5976a5f154753536", - "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456", - "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456"}) + self.assertEqual( + {call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list}, + { + "macos-xcode-ccache-refs/pull/226/merge-6c8d283f5bc894af8dfc295e5976a5f154753123", + "ubuntu-24.04-ccache-refs/pull/227/merge-9d6d283f5bc894af8dfc295e5976a5f1b46649c4", + "ubuntu-24.04-ccache-refs/pull/227/merge-1d6d283f5bc894af8dfc295e5976a5f154753487", + "macos-xcode-ccache-refs/pull/227/merge-2d6d283f5bc894af8dfc295e5976a5f154753536", + "macos-xcode-ccache-refs/heads/MINIFICPP-9999-9d5e183f5bc894af8dfc295e5976a5f1b4664456", + "ubuntu-24.04-all-clang-ccache-refs/heads/main-1d4d283f5bc894af8dfc295e5976a5f1b4664456", + }, + ) def test_cache_cleanup_with_zero_action_caches(self): cleaner = GithubActionsCacheCleaner("mytoken", "githubuser/nifi-minifi-cpp") cleaner.github_request_sender = self.create_empty_caches_mock_github_request_sender() cleaner.remove_obsolete_cache_entries() - self.assertEqual(set([call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list]), set()) + self.assertEqual( + {call[0][0] for call in cleaner.github_request_sender.delete_cache.call_args_list}, + set(), + ) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a812f76a3d..c9d2b132b6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -660,19 +660,19 @@ jobs: name: minifi_rs_behave path: minifi_rust/minifi_rs_behave/output linters: - name: "C++ lint + Shellcheck + Flake8 + Cargo fmt check + Clippy check" + name: "C++ lint + Shellcheck + Ruff + Cargo fmt check + Clippy check" if: >- github.event_name != 'push' || github.repository != 'apache/nifi-minifi-cpp' || github.ref == 'refs/heads/main' - runs-on: ubuntu-22.04-arm + runs-on: ubuntu-24.04-arm timeout-minutes: 15 steps: - id: checkout uses: actions/checkout@v6 - id: install_deps - run: sudo apt update && sudo apt install -y flake8 + run: pipx install ruff - id: cpp_lint name: C++ linter @@ -684,10 +684,10 @@ jobs: continue-on-error: true run: ./run_shellcheck.sh . - - id: flake8_check - name: Flake8 check + - id: python_check + name: Python check continue-on-error: true - run: ./run_flake8.sh . + run: ruff check && ruff format --check - id: cargo_fmt_check name: Cargo fmt check @@ -702,7 +702,7 @@ jobs: working-directory: minifi_rust - name: Check Linter Statuses - if: steps.cpp_lint.outcome == 'failure' || steps.shellcheck.outcome == 'failure' || steps.flake8_check.outcome == 'failure' || steps.cargo_fmt_check.outcome == 'failure' || steps.clippy_check.outcome == 'failure' + if: steps.cpp_lint.outcome == 'failure' || steps.shellcheck.outcome == 'failure' || steps.python_check.outcome == 'failure' || steps.cargo_fmt_check.outcome == 'failure' || steps.clippy_check.outcome == 'failure' run: | echo "One or more linters failed. Failing the workflow." exit 1 diff --git a/CMakeLists.txt b/CMakeLists.txt index 2d0903c2ec..8523e368ce 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -832,9 +832,6 @@ set_target_properties(linter PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD TRUE) if(NOT WIN32) add_custom_target(shellcheck COMMAND ${CMAKE_SOURCE_DIR}/run_shellcheck.sh ${CMAKE_SOURCE_DIR}) - -add_custom_target(flake8 - COMMAND ${CMAKE_SOURCE_DIR}/run_flake8.sh ${CMAKE_SOURCE_DIR}) endif(NOT WIN32) feature_summary(WHAT ALL FILENAME ${CMAKE_BINARY_DIR}/all.log) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a1d25c6ca7..a0338b098b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -55,14 +55,28 @@ it doesn't contradict any of the Google Style Guide or the above exceptions. C++ is a powerful language and "with great power comes great responsibility". Please aim for simple, readable and maintainable solutions and avoid footguns. Be open for respectful debate in pull request reviews. +### Shell scripts + Shell script files shall follow the guidelines and best practices defined by the [shellcheck](https://github.com/koalaman/shellcheck) analysis tool. New contributions are expected to pass the shellcheck analysis as part of the verification process. If a shellcheck requested change is unfeasible it shall be disabled on per-line basis and will be subjected to review. For more information on an issue please check the [shellcheck wiki page](https://github.com/koalaman/shellcheck/wiki). -You can run shellcheck by invoking the shellcheck cmake target, e.g. `make shellcheck`. +You can run shellcheck by invoking the shellcheck cmake target. +```bash +make shellcheck +``` -Python script files shall follow the PEP8 guidelines and best practices. The project includes [flake8](https://flake8.pycqa.org/en/latest/) checks +### Python +Python script files shall follow the PEP8 guidelines and best practices. The project includes [ruff](https://docs.astral.sh/ruff/) checks as part of the verification process, that is applied to all new contributions. +You can run the linter on Python code via: +```bash +ruff check +``` +You can also auto-format the code via +```bash +ruff format +``` ## Issues and Pull Requests @@ -127,4 +141,3 @@ register_extension(minifi-kubernetes-extensions "KUBERNETES EXTENSIONS" KUBERNET # the next three arguments are used for documentation purposes # the fifth optional argument designates the directory of the extension's tests ``` - diff --git a/behave_framework/src/minifi_behave/containers/container_linux.py b/behave_framework/src/minifi_behave/containers/container_linux.py index da0dcbc0ab..32fc541195 100644 --- a/behave_framework/src/minifi_behave/containers/container_linux.py +++ b/behave_framework/src/minifi_behave/containers/container_linux.py @@ -23,22 +23,31 @@ import tarfile import tempfile import uuid +from typing import TYPE_CHECKING + from docker.models.networks import Network from minifi_behave.containers.container_protocol import ContainerProtocol from minifi_behave.containers.directory import Directory from minifi_behave.containers.file import File from minifi_behave.containers.host_file import HostFile -from typing import TYPE_CHECKING import docker +logger = logging.getLogger(__name__) + if TYPE_CHECKING: from minifi_behave.core.minifi_test_context import MinifiTestContext class LinuxContainer(ContainerProtocol): - def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None, - entrypoint: str | None = None): + def __init__( + self, + image_name: str, + container_name: str, + network: Network, + command: str | None = None, + entrypoint: str | None = None, + ): super().__init__() self.image_name: str = image_name self.container_name: str = container_name @@ -64,13 +73,13 @@ def add_host_file(self, host_path: str, container_path: str, mode: str = "ro"): def add_file_to_running_container(self, content: str, path: str): if not self.container: - logging.error("Container is not running. Cannot add file.") + logger.error("Container is not running. Cannot add file.") raise RuntimeError("Container is not running. Cannot add file.") mkdir_command = f"mkdir -p {shlex.quote(path)}" exit_code, output = self.exec_run(mkdir_command) if exit_code != 0: - logging.error(f"Error creating directory '{path}' in container: {output}") + logger.error(f"Error creating directory '{path}' in container: {output}") raise RuntimeError(f"Error creating directory '{path}' in container: {output}") full_path = os.path.join(path, str(uuid.uuid4())) @@ -78,7 +87,7 @@ def add_file_to_running_container(self, content: str, path: str): pipe_command = f"printf %s {shlex.quote(content)} > {shlex.quote(tmp_path)} && mv {shlex.quote(tmp_path)} {shlex.quote(full_path)}" exit_code, output = self.exec_run(f"sh -c {shlex.quote(pipe_command)}") if exit_code != 0: - logging.error(f"Error adding file to running container: {output}") + logger.error(f"Error adding file to running container: {output}") raise RuntimeError(f"Error adding file to running container: {output}") def _write_content_to_file(self, filepath: str, permissions: int | None, content: str | bytes): @@ -105,35 +114,42 @@ def _configure_volumes_of_container_dirs(self): self._write_content_to_file(file_path, None, content) self.volumes[temp_path] = {"bind": directory.path, "mode": directory.mode} - def is_deployed(self) -> bool: - return self.container is not None - def deploy(self, context: MinifiTestContext | None) -> bool: if self.is_deployed(): - logging.info(f"Container '{self.container_name}' is already deployed.") + logger.info(f"Container '{self.container_name}' is already deployed.") return True self._temp_dir = tempfile.TemporaryDirectory() self._configure_volumes_of_container_files() self._configure_volumes_of_container_dirs() for host_file in self.host_files: - self.volumes[host_file.host_path] = {"bind": host_file.container_path, "mode": host_file.mode} + self.volumes[host_file.host_path] = { + "bind": host_file.container_path, + "mode": host_file.mode, + } try: existing_container = self.client.containers.get(self.container_name) - logging.warning(f"Found existing container '{self.container_name}'. Removing it first.") + logger.warning(f"Found existing container '{self.container_name}'. Removing it first.") existing_container.remove(force=True) except docker.errors.NotFound: pass try: - logging.info(f"Creating and starting container '{self.container_name}'...") - self.container = self.client.containers.run(image=self.image_name, name=self.container_name, - ports=self.ports, environment=self.environment, - volumes=self.volumes, network=self.network.name, - command=self.command, entrypoint=self.entrypoint, - user=self.user, detach=True) + logger.info(f"Creating and starting container '{self.container_name}'...") + self.container = self.client.containers.run( + image=self.image_name, + name=self.container_name, + ports=self.ports, + environment=self.environment, + volumes=self.volumes, + network=self.network.name, + command=self.command, + entrypoint=self.entrypoint, + user=self.user, + detach=True, + ) except Exception as e: - logging.error(f"Error starting container: {e}") + logger.error(f"Error starting container: {e}") raise return True @@ -141,32 +157,32 @@ def start(self): if self.container: self.container.start() else: - logging.error("Container does not exist. Cannot start.") + logger.error("Container does not exist. Cannot start.") def stop(self): if self.container: self.container.stop() else: - logging.error("Container does not exist. Cannot stop.") + logger.error("Container does not exist. Cannot stop.") def kill(self): if self.container: self.container.kill() else: - logging.error("Container does not exist. Cannot kill.") + logger.error("Container does not exist. Cannot kill.") def restart(self): if self.container: self.container.restart() else: - logging.error("Container does not exist. Cannot restart.") + logger.error("Container does not exist. Cannot restart.") def clean_up(self): if self.container: try: self.container.remove(force=True) except Exception as e: - logging.error(f"Error cleaning up container '{self.container_name}': {e}") + logger.error(f"Error cleaning up container '{self.container_name}': {e}") def exec_run(self, command) -> tuple[int | None, str]: if self.container: @@ -177,12 +193,14 @@ def exec_run(self, command) -> tuple[int | None, str]: def nonempty_dir_exists(self, directory_path: str) -> bool: if not self.container: return False - dir_exists_exit_code, dir_exists_output = self.exec_run( - "sh -c {}".format(shlex.quote(f"test -d {directory_path}"))) + dir_exists_exit_code, _dir_exists_output = self.exec_run( + "sh -c {}".format(shlex.quote(f"test -d {directory_path}")) + ) if dir_exists_exit_code != 0: return False - dir_not_empty_ec, dir_not_empty_output = self.exec_run( - "sh -c {}".format(shlex.quote(f'[ "$(ls -A {directory_path})" ]'))) + dir_not_empty_ec, _dir_not_empty_output = self.exec_run( + "sh -c {}".format(shlex.quote(f'[ "$(ls -A {directory_path})" ]')) + ) return dir_not_empty_ec == 0 def directory_contains_empty_file(self, directory_path: str) -> bool: @@ -213,27 +231,31 @@ def directory_contains_file_with_regex(self, directory_path: str, regex_str: str safe_dir_path = shlex.quote(directory_path) safe_regex_str = shlex.quote(regex_str) - command = (f"find {safe_dir_path} -maxdepth 1 -type f -print0 | " - f"xargs -0 -r grep -l -E -- {safe_regex_str}") + command = f"find {safe_dir_path} -maxdepth 1 -type f -print0 | xargs -0 -r grep -l -E -- {safe_regex_str}" - exit_code, output = self.exec_run("sh -c {}".format(shlex.quote(command))) + exit_code, output = self.exec_run(f"sh -c {shlex.quote(command)}") if exit_code != 0: - logging.debug("While looking for regex %s in directory %s, grep returned exit code %d, output: %s", - regex_str, directory_path, exit_code, output) + logger.debug( + "While looking for regex %s in directory %s, grep returned exit code %d, output: %s", + regex_str, + directory_path, + exit_code, + output, + ) return exit_code == 0 def path_with_content_exists(self, path: str, content: str) -> bool: count_command = f"sh -c 'cat {path} | grep \"^{content}$\" | wc -l'" exit_code, output = self.exec_run(count_command) if exit_code != 0: - logging.error(f"Error running command '{count_command}': {output}") + logger.error(f"Error running command '{count_command}': {output}") return False try: file_count = int(output.strip()) except (ValueError, IndexError): - logging.error(f"Error parsing output '{output}' from command '{count_command}'") + logger.error(f"Error parsing output '{output}' from command '{count_command}'") return False return file_count == 1 @@ -245,36 +267,36 @@ def directory_has_single_file_with_content(self, directory_path: str, expected_c exit_code, output = self.exec_run(count_command) if exit_code != 0: - logging.error(f"Error running command '{count_command}': {output}") + logger.error(f"Error running command '{count_command}': {output}") return False try: file_count = int(output.strip()) except (ValueError, IndexError): - logging.error(f"Error parsing output '{output}' from command '{count_command}'") + logger.error(f"Error parsing output '{output}' from command '{count_command}'") return False if file_count != 1: - logging.error(f"{directory_path} has too many or too few ({file_count}) files") + logger.error(f"{directory_path} has too many or too few ({file_count}) files") return False content_command = f"sh -c 'cat {directory_path}/*'" exit_code, output = self.exec_run(content_command) if exit_code != 0: - logging.error(f"Error running command '{content_command}': {output}") + logger.error(f"Error running command '{content_command}': {output}") return False actual_content = output.strip() - logging.debug(f"Comparing: '{actual_content}' vs {expected_content}") + logger.debug(f"Comparing: '{actual_content}' vs {expected_content}") return actual_content == expected_content.strip() def get_logs(self) -> str: - logging.debug("Getting logs from container '%s'", self.container_name) + logger.debug("Getting logs from container '%s'", self.container_name) if not self.container: return "" logs_as_bytes = self.container.logs() - return logs_as_bytes.decode('utf-8') + return logs_as_bytes.decode("utf-8") @property def exited(self) -> bool: @@ -282,7 +304,7 @@ def exited(self) -> bool: return False try: self.container.reload() - return self.container.status == 'exited' + return self.container.status == "exited" except docker.errors.NotFound: self.container = None return False @@ -291,39 +313,39 @@ def exited(self) -> bool: def get_number_of_files(self, directory_path: str) -> int: if not self.container: - logging.warning("Container not running") + logger.warning("Container not running") return -1 if not self.nonempty_dir_exists(directory_path): - logging.warning(f"Container directory does not exist: {directory_path}") + logger.warning(f"Container directory does not exist: {directory_path}") return 0 count_command = f"sh -c 'find {directory_path} -maxdepth 1 -type f | wc -l'" exit_code, output = self.exec_run(count_command) if exit_code != 0: - logging.error(f"Error running command '{count_command}': {output}") + logger.error(f"Error running command '{count_command}': {output}") return -1 try: file_count = int(output.strip()) - logging.debug(f"Number of files in '{directory_path}': {file_count}") + logger.debug(f"Number of files in '{directory_path}': {file_count}") return file_count except (ValueError, IndexError): - logging.error(f"Error parsing output '{output}' from command '{count_command}'") + logger.error(f"Error parsing output '{output}' from command '{count_command}'") return -1 def _get_contents_of_all_files_in_directory(self, directory_path: str) -> list[str] | None: safe_dir_path = shlex.quote(directory_path) list_files_command = f"find {safe_dir_path} -mindepth 1 -maxdepth 1 -type f -print0" - exit_code, output = self.exec_run(f"sh -c \"{list_files_command}\"") + exit_code, output = self.exec_run(f'sh -c "{list_files_command}"') if exit_code != 0: - logging.error(f"Error running command '{list_files_command}': {output}") + logger.error(f"Error running command '{list_files_command}': {output}") return None - actual_filepaths = [path for path in output.split('\0') if path] + actual_filepaths = [path for path in output.split("\0") if path] actual_file_contents = [] for path in actual_filepaths: @@ -334,7 +356,7 @@ def _get_contents_of_all_files_in_directory(self, directory_path: str) -> list[s if exit_code != 0: error_message = f"Command to read file '{path}' failed with exit code {exit_code}" - logging.error(error_message) + logger.error(error_message) return None actual_file_contents.append(content) @@ -350,7 +372,7 @@ def _verify_file_contents_in_running_container(self, directory_path: str, expect return False if len(actual_file_contents) != len(expected_contents): - logging.debug(f"Expected {len(expected_contents)} files, but found {len(actual_file_contents)}") + logger.debug(f"Expected {len(expected_contents)} files, but found {len(actual_file_contents)}") return False return sorted(actual_file_contents) == sorted(expected_contents) @@ -375,17 +397,17 @@ def _extract_directory_from_container(self, directory_path: str, temp_dir: str) bits, _ = self.container.get_archive(directory_path) temp_tar_path = os.path.join(temp_dir, "archive.tar") - with open(temp_tar_path, 'wb') as f: - for chunk in bits: - f.write(chunk) + with open(temp_tar_path, "wb") as f: + f.writelines(bits) with tarfile.open(temp_tar_path) as tar: tar.extractall(path=temp_dir) - return os.path.join(temp_dir, os.path.basename(directory_path.strip('/'))) + return os.path.join(temp_dir, os.path.basename(directory_path.strip("/"))) except Exception as e: - logging.error( - f"Error extracting files from directory path '{directory_path}' from container '{self.container_name}': {e}") + logger.error( + f"Error extracting files from directory path '{directory_path}' from container '{self.container_name}': {e}" + ) return None def _read_files_from_directory(self, directory_path: str) -> list[str] | None: @@ -393,11 +415,11 @@ def _read_files_from_directory(self, directory_path: str) -> list[str] | None: file_contents = [] for entry in os.scandir(directory_path): if entry.is_file(): - with open(entry.path, 'r') as f: + with open(entry.path, "r") as f: file_contents.append(f.read()) return file_contents except Exception as e: - logging.error(f"Error reading extracted files: {e}") + logger.error(f"Error reading extracted files: {e}") return None def verify_file_contents(self, directory_path: str, expected_contents: list[str]) -> bool: @@ -412,38 +434,38 @@ def verify_file_contents(self, directory_path: str, expected_contents: list[str] def log_app_output(self) -> bool: logs = self.get_logs() - logging.info("Logs of container '%s':", self.container_name) + logger.info("Logs of container '%s':", self.container_name) for line in logs.splitlines(): - logging.info(line) + logger.info(line) return False def verify_path_with_json_content(self, directory_path: str, expected_str: str) -> bool: if not self.container or not self.nonempty_dir_exists(directory_path): - logging.warning(f"Container not running or directory does not exist: {directory_path}") + logger.warning(f"Container not running or directory does not exist: {directory_path}") return False count_command = f"sh -c 'find {directory_path} -maxdepth 1 -type f | wc -l'" exit_code, output = self.exec_run(count_command) if exit_code != 0: - logging.error(f"Error running command '{count_command}': {output}") + logger.error(f"Error running command '{count_command}': {output}") return False try: file_count = int(output.strip()) except (ValueError, IndexError): - logging.error(f"Error parsing output '{output}' from command '{count_command}'") + logger.error(f"Error parsing output '{output}' from command '{count_command}'") return False if file_count != 1: - logging.error(f"{directory_path} has too many or too few ({file_count}) files") + logger.error(f"{directory_path} has too many or too few ({file_count}) files") return False content_command = f"sh -c 'cat {directory_path}/*'" exit_code, output = self.exec_run(content_command) if exit_code != 0: - logging.error(f"Error running command '{content_command}': {output}") + logger.error(f"Error running command '{content_command}': {output}") return False actual_content = output.strip() @@ -454,7 +476,7 @@ def verify_path_with_json_content(self, directory_path: str, expected_str: str) def directory_contains_file_with_json_content(self, directory_path: str, expected_content: str) -> bool: if not self.container or not self.nonempty_dir_exists(directory_path): - logging.warning(f"Container not running or directory does not exist: {directory_path}") + logger.warning(f"Container not running or directory does not exist: {directory_path}") return False actual_file_contents = self._get_contents_of_all_files_in_directory(directory_path) @@ -467,9 +489,9 @@ def directory_contains_file_with_json_content(self, directory_path: str, expecte expected_json = json.loads(expected_content) if actual_json == expected_json: return True - logging.warning(f"File content does not match expected JSON: {file_content}") + logger.warning(f"File content does not match expected JSON: {file_content}") except json.JSONDecodeError: - logging.error("Error decoding JSON content from file.") + logger.error("Error decoding JSON content from file.") continue return False @@ -478,21 +500,18 @@ def directory_contains_file_with_minimum_size(self, directory_path: str, expecte if not self.container or not self.nonempty_dir_exists(directory_path): return False - command = f"find \"{directory_path}\" -maxdepth 1 -type f -size +{expected_size}c" + command = f'find "{directory_path}" -maxdepth 1 -type f -size +{expected_size}c' exit_code, output = self.exec_run(command) if exit_code != 0: - logging.error(f"Error running command to get file sizes: {output}") + logger.error(f"Error running command to get file sizes: {output}") return False - if len(output.strip()) > 0: - return True - - return False + return len(output.strip()) > 0 def get_memory_usage(self) -> int | None: - exit_code, output = self.exec_run(["awk", "/VmRSS.*kB/ { printf \"%d\", $2 }", "/proc/1/status"]) + exit_code, output = self.exec_run(["awk", '/VmRSS.*kB/ { printf "%d", $2 }', "/proc/1/status"]) if exit_code != 0: return None memory_usage_in_bytes = int(output) * 1024 - logging.info(f"{self.container_name} memory usage: {memory_usage_in_bytes} bytes") + logger.info(f"{self.container_name} memory usage: {memory_usage_in_bytes} bytes") return memory_usage_in_bytes diff --git a/behave_framework/src/minifi_behave/containers/container_protocol.py b/behave_framework/src/minifi_behave/containers/container_protocol.py index 2b2fdcc721..cc167c33b3 100644 --- a/behave_framework/src/minifi_behave/containers/container_protocol.py +++ b/behave_framework/src/minifi_behave/containers/container_protocol.py @@ -27,36 +27,25 @@ class ContainerProtocol(Protocol): files: list[File] host_files: list[HostFile] - def deploy(self, context) -> bool: - ... + def deploy(self, context) -> bool: ... - def clean_up(self): - ... + def clean_up(self): ... - def exec_run(self, command): - ... + def exec_run(self, command): ... - def directory_contains_file_with_content(self, directory_path: str, expected_content: str) -> bool: - ... + def directory_contains_file_with_content(self, directory_path: str, expected_content: str) -> bool: ... - def directory_contains_file_with_regex(self, directory_path: str, regex_str: str) -> bool: - ... + def directory_contains_file_with_regex(self, directory_path: str, regex_str: str) -> bool: ... - def path_with_content_exists(self, path: str, content: str) -> bool: - ... + def path_with_content_exists(self, path: str, content: str) -> bool: ... - def get_logs(self) -> str: - ... + def get_logs(self) -> str: ... @property - def exited(self) -> bool: - ... + def exited(self) -> bool: ... - def get_number_of_files(self, directory_path: str) -> int: - ... + def get_number_of_files(self, directory_path: str) -> int: ... - def verify_file_contents(self, directory_path: str, expected_contents: list[str]) -> bool: - ... + def verify_file_contents(self, directory_path: str, expected_contents: list[str]) -> bool: ... - def log_app_output(self) -> bool: - ... + def log_app_output(self) -> bool: ... diff --git a/behave_framework/src/minifi_behave/containers/container_windows.py b/behave_framework/src/minifi_behave/containers/container_windows.py index 87b0a6243b..5117562b70 100644 --- a/behave_framework/src/minifi_behave/containers/container_windows.py +++ b/behave_framework/src/minifi_behave/containers/container_windows.py @@ -15,29 +15,39 @@ # limitations under the License. # from __future__ import annotations + +import base64 +import io import logging import os -import tempfile -import base64 import tarfile -import io +import tempfile from typing import TYPE_CHECKING -import docker -from docker.models.networks import Network from docker.models.containers import Container - +from docker.models.networks import Network from minifi_behave.containers.container_protocol import ContainerProtocol from minifi_behave.containers.directory import Directory from minifi_behave.containers.file import File from minifi_behave.containers.host_file import HostFile +import docker + +logger = logging.getLogger(__name__) + if TYPE_CHECKING: from minifi_behave.core.minifi_test_context import MinifiTestContext class WindowsContainer(ContainerProtocol): - def __init__(self, image_name: str, container_name: str, network: Network, command: str | None = None, entrypoint: str | None = None): + def __init__( + self, + image_name: str, + container_name: str, + network: Network, + command: str | None = None, + entrypoint: str | None = None, + ): super().__init__() self.image_name: str = image_name self.container_name: str = container_name @@ -57,8 +67,7 @@ def __init__(self, image_name: str, container_name: str, network: Network, comma def _normalize_path(self, path: str) -> str: clean_path = path.strip().replace("/", "\\") - if clean_path.startswith("\\"): - clean_path = clean_path[1:] + clean_path = clean_path.removeprefix("\\") # If it doesn't already have a drive letter, assume C: if ":" not in clean_path: @@ -78,28 +87,31 @@ def deploy(self, context: MinifiTestContext | None) -> bool: for file_name, content in directory.files.items(): file_path = os.path.join(temp_subdir, file_name) with open(file_path, "w", encoding="utf-8") as temp_file: - logging.info(f"writing content into {temp_file.name}") + logger.info(f"writing content into {temp_file.name}") temp_file.write(content) container_bind_path = self._normalize_path(directory.path) self.volumes[temp_subdir] = { "bind": container_bind_path, - "mode": directory.mode + "mode": directory.mode, } for host_file in self.host_files: container_bind_path = self._normalize_path(host_file.container_path) - self.volumes[host_file.host_path] = {"bind": container_bind_path, "mode": host_file.mode} + self.volumes[host_file.host_path] = { + "bind": container_bind_path, + "mode": host_file.mode, + } try: existing_container = self.client.containers.get(self.container_name) - logging.warning(f"Found existing container '{self.container_name}'. Removing it first.") + logger.warning(f"Found existing container '{self.container_name}'. Removing it first.") existing_container.remove(force=True) except docker.errors.NotFound: pass try: - logging.info(f"Creating and starting container '{self.container_name}'...") + logger.info(f"Creating and starting container '{self.container_name}'...") self.container = self.client.containers.run( image=self.image_name, name=self.container_name, @@ -110,14 +122,14 @@ def deploy(self, context: MinifiTestContext | None) -> bool: command=self.command, entrypoint=self.entrypoint, detach=True, - tty=False + tty=False, ) for file in self.files: self._copy_content_to_container(file.content, file.path) except Exception as e: - logging.error(f"Error starting container: {e}") + logger.error(f"Error starting container: {e}") self.clean_up() raise return True @@ -133,9 +145,9 @@ def _copy_content_to_container(self, content: str | bytes, target_path: str): self._run_powershell(f"New-Item -ItemType Directory -Force -Path '{dir_name}'") tar_stream = io.BytesIO() - with tarfile.open(fileobj=tar_stream, mode='w') as tar: + with tarfile.open(fileobj=tar_stream, mode="w") as tar: if isinstance(content, str): - encoded_data = content.encode('utf-8') + encoded_data = content.encode("utf-8") else: encoded_data = content tarinfo = tarfile.TarInfo(name=file_name) @@ -151,7 +163,7 @@ def clean_up(self): try: self._temp_dir.cleanup() except Exception as e: - logging.warning(f"Failed to cleanup temp dir: {e}") + logger.warning(f"Failed to cleanup temp dir: {e}") finally: self._temp_dir = None @@ -161,16 +173,16 @@ def clean_up(self): except docker.errors.NotFound: pass except Exception as e: - logging.warning(f"Failed to remove container: {e}") + logger.warning(f"Failed to remove container: {e}") finally: self.container = None def exec_run(self, command: str | list) -> tuple[int | None, str]: - logging.debug(f"Running command: {command}") + logger.debug(f"Running command: {command}") if self.container: (code, output) = self.container.exec_run(command, detach=False) - decoded_output = output.decode("utf-8", errors='replace') - logging.debug(f"Result {code}, output: {decoded_output}") + decoded_output = output.decode("utf-8", errors="replace") + logger.debug(f"Result {code}, output: {decoded_output}") return code, decoded_output return None, "Container not running." @@ -178,9 +190,15 @@ def _run_powershell(self, ps_script: str) -> tuple[int | None, str]: if not self.container: return None, "Container not running" - encoded_command = base64.b64encode(ps_script.encode('utf_16_le')).decode('utf-8') + encoded_command = base64.b64encode(ps_script.encode("utf_16_le")).decode("utf-8") - cmd_parts = ["powershell", "-NonInteractive", "-NoProfile", "-EncodedCommand", encoded_command] + cmd_parts = [ + "powershell", + "-NonInteractive", + "-NoProfile", + "-EncodedCommand", + encoded_command, + ] return self.exec_run(cmd_parts) @@ -249,7 +267,7 @@ def path_with_content_exists(self, path: str, content: str) -> bool: exit_code, output = self._run_powershell(ps_script) if exit_code != 0: - logging.debug(f"path_with_content_exists failed for {win_path}. Output: {output}") + logger.debug(f"path_with_content_exists failed for {win_path}. Output: {output}") return exit_code == 0 @@ -269,22 +287,22 @@ def directory_has_single_file_with_content(self, directory_path: str, expected_c exit_code, output = self._run_powershell(ps_script) if exit_code != 0: - logging.debug(f"Check for single file failed in {win_path}. Output: {output}") + logger.debug(f"Check for single file failed in {win_path}. Output: {output}") return exit_code == 0 def get_logs(self) -> str: - logging.debug("Getting logs from container '%s'", self.container_name) + logger.debug("Getting logs from container '%s'", self.container_name) if not self.container: return "" logs_as_bytes = self.container.logs() - return logs_as_bytes.decode('utf-8', errors='replace') + return logs_as_bytes.decode("utf-8", errors="replace") def log_app_output(self) -> bool: logs = self.get_logs() - logging.info("Logs of container '%s':", self.container_name) + logger.info("Logs of container '%s':", self.container_name) for line in logs.splitlines(): - logging.info(line) + logger.info(line) return False @property @@ -293,7 +311,7 @@ def exited(self) -> bool: return False try: self.container.reload() - return self.container.status == 'exited' + return self.container.status == "exited" except docker.errors.NotFound: self.container = None return False @@ -310,7 +328,7 @@ def get_number_of_files(self, directory_path: str) -> int: exit_code, output = self._run_powershell(ps_script) if exit_code != 0: - logging.error(f"Error counting files in '{win_path}': {output}") + logger.error(f"Error counting files in '{win_path}': {output}") return -1 try: @@ -328,7 +346,7 @@ def verify_file_contents(self, directory_path: str, expected_contents: list[str] exit_code, output = self._run_powershell(ps_list) if exit_code != 0: - logging.error(f"Error listing files in '{win_path}': {output}") + logger.error(f"Error listing files in '{win_path}': {output}") return False actual_filepaths = [path.strip() for path in output.splitlines() if path.strip()] @@ -348,9 +366,7 @@ def verify_file_contents(self, directory_path: str, expected_contents: list[str] return False actual_file_contents.append(content) - normalized_expected = [ - s.strip().replace("\r\n", "\n").replace("\r", "\n") for s in expected_contents - ] + normalized_expected = [s.strip().replace("\r\n", "\n").replace("\r", "\n") for s in expected_contents] return sorted(actual_file_contents) == sorted(normalized_expected) @@ -369,7 +385,7 @@ def nonempty_dir_exists(self, directory_path: str) -> bool: exit_code, output = self._run_powershell(ps_script) if exit_code != 0: - logging.error(f"Error running command for nonempty_dir_exists: {output}") + logger.error(f"Error running command for nonempty_dir_exists: {output}") return False return True @@ -379,18 +395,12 @@ def directory_contains_file_with_minimum_size(self, directory_path: str, expecte return False win_path = self._normalize_path(directory_path) - ps_script = ( - f"Get-ChildItem -LiteralPath '{win_path}' -File " - f"| Where-Object {{ $_.Length -gt {expected_size} }}" - ) + ps_script = f"Get-ChildItem -LiteralPath '{win_path}' -File | Where-Object {{ $_.Length -gt {expected_size} }}" exit_code, output = self._run_powershell(ps_script) if exit_code != 0: - logging.error(f"Error running command to get file sizes: {output}") + logger.error(f"Error running command to get file sizes: {output}") return False - if output and len(output.strip()) > 0: - return True - - return False + return bool(output and len(output.strip()) > 0) diff --git a/behave_framework/src/minifi_behave/containers/directory.py b/behave_framework/src/minifi_behave/containers/directory.py index 47462ef91e..4628b0413b 100644 --- a/behave_framework/src/minifi_behave/containers/directory.py +++ b/behave_framework/src/minifi_behave/containers/directory.py @@ -15,6 +15,7 @@ # limitations under the License. # + class Directory: def __init__(self, path, files: dict[str, str | bytes] | None = None, mode="rw"): self.path = path diff --git a/behave_framework/src/minifi_behave/containers/docker_image_builder.py b/behave_framework/src/minifi_behave/containers/docker_image_builder.py index 1e5d5e602f..4a172cbae5 100644 --- a/behave_framework/src/minifi_behave/containers/docker_image_builder.py +++ b/behave_framework/src/minifi_behave/containers/docker_image_builder.py @@ -19,12 +19,21 @@ import os import tempfile -import docker from docker.models.images import Image +import docker + +logger = logging.getLogger(__name__) + class DockerImageBuilder: - def __init__(self, image_tag: str, dockerfile_content: str | None = None, files_on_context: dict[str, bytes] | None = None, build_context_path: str | None = None): + def __init__( + self, + image_tag: str, + dockerfile_content: str | None = None, + files_on_context: dict[str, bytes] | None = None, + build_context_path: str | None = None, + ): if not dockerfile_content and not build_context_path: raise ValueError("Either 'dockerfile_content' or 'build_context_path' must be provided.") if dockerfile_content and build_context_path: @@ -43,27 +52,29 @@ def build(self) -> Image: if self.dockerfile_content: self._temp_dir = tempfile.TemporaryDirectory() context_path = self._temp_dir.name - dockerfile_path = os.path.join(context_path, 'Dockerfile') - with open(dockerfile_path, 'w') as f: + dockerfile_path = os.path.join(context_path, "Dockerfile") + with open(dockerfile_path, "w") as f: f.write(self.dockerfile_content) if self.files_on_context: for filename, content in self.files_on_context.items(): file_path = os.path.join(context_path, filename) - with open(file_path, 'wb') as f: + with open(file_path, "wb") as f: f.write(content) - logging.info(f"Building Docker image '{self.image_tag}' from context '{context_path}'...") + logger.info(f"Building Docker image '{self.image_tag}' from context '{context_path}'...") try: - self.image, build_logs = self.client.images.build(path=context_path, tag=self.image_tag, rm=True, forcerm=True) + self.image, build_logs = self.client.images.build( + path=context_path, tag=self.image_tag, rm=True, forcerm=True + ) for log_line in build_logs: - logging.debug(log_line.get('stream', '').strip()) - logging.info(f"Successfully built image '{self.image_tag}' (ID: {self.image.short_id})") + logger.debug(log_line.get("stream", "").strip()) + logger.info(f"Successfully built image '{self.image_tag}' (ID: {self.image.short_id})") return self.image except docker.errors.BuildError as e: - logging.error(f"Failed to build image '{self.image_tag}'. Build logs:") + logger.error(f"Failed to build image '{self.image_tag}'. Build logs:") for log_line in e.build_log: - logging.error(log_line.get('stream', '').strip()) + logger.error(log_line.get("stream", "").strip()) raise finally: if self._temp_dir: @@ -71,18 +82,18 @@ def build(self) -> Image: def remove_image(self): if not self.image: - logging.warning(f"No image object to remove for tag '{self.image_tag}'. Trying to find by tag.") + logger.warning(f"No image object to remove for tag '{self.image_tag}'. Trying to find by tag.") try: self.image = self.client.images.get(self.image_tag) except docker.errors.ImageNotFound: - logging.info(f"Image '{self.image_tag}' not found, cleanup not needed.") + logger.info(f"Image '{self.image_tag}' not found, cleanup not needed.") return - logging.info(f"Removing dynamically built image '{self.image_tag}'...") + logger.info(f"Removing dynamically built image '{self.image_tag}'...") try: self.client.images.remove(image=self.image.id, force=True) - logging.info("Image removed successfully.") + logger.info("Image removed successfully.") except docker.errors.ImageNotFound: - logging.info("Image was already removed.") + logger.info("Image was already removed.") except docker.errors.APIError as e: - logging.error(f"Error removing image: {e}") + logger.error(f"Error removing image: {e}") diff --git a/behave_framework/src/minifi_behave/containers/file.py b/behave_framework/src/minifi_behave/containers/file.py index ee50af95e9..6df14a92e9 100644 --- a/behave_framework/src/minifi_behave/containers/file.py +++ b/behave_framework/src/minifi_behave/containers/file.py @@ -15,6 +15,7 @@ # limitations under the License. # + class File: def __init__(self, path, content: str | bytes, mode="rw", permissions=None): self.path = path diff --git a/behave_framework/src/minifi_behave/containers/host_file.py b/behave_framework/src/minifi_behave/containers/host_file.py index 1e046f54c6..f68844fc81 100644 --- a/behave_framework/src/minifi_behave/containers/host_file.py +++ b/behave_framework/src/minifi_behave/containers/host_file.py @@ -15,6 +15,7 @@ # limitations under the License. # + class HostFile: def __init__(self, path, host_path, mode="ro"): self.container_path = path diff --git a/behave_framework/src/minifi_behave/containers/http_proxy_container.py b/behave_framework/src/minifi_behave/containers/http_proxy_container.py index 4063363e9b..19bd5a12d5 100644 --- a/behave_framework/src/minifi_behave/containers/http_proxy_container.py +++ b/behave_framework/src/minifi_behave/containers/http_proxy_container.py @@ -19,15 +19,16 @@ from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.helpers import wait_for_condition, retry_check -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.ssl_utils import make_server_cert, dump_cert, dump_key from minifi_behave.containers.file import File +from minifi_behave.core.helpers import retry_check, wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert class HttpProxy(LinuxContainer): def __init__(self, test_context: MinifiTestContext): - dockerfile = dedent("""\ + dockerfile = dedent( + """\ FROM ubuntu:24.04 RUN apt -y update && apt install -y squid-openssl apache2-utils RUN htpasswd -b -c /etc/squid/.squid_users {proxy_username} {proxy_password} @@ -43,19 +44,33 @@ def __init__(self, test_context: MinifiTestContext): echo 'max_filedescriptors 1024' >> /etc/squid/squid.conf && \ echo 'https_port {proxy_ssl_port} tls-cert=/etc/squid/certs/squid-cert.pem tls-key=/etc/squid/certs/squid-key.pem' >> /etc/squid/squid.conf ENTRYPOINT ["/bin/sh", "-c", "squid -N -f /etc/squid/squid.conf & tail -F --pid=$! /var/log/squid/cache.log /var/log/squid/access.log"] - """.format(proxy_username='admin', proxy_password='test101', - proxy_port='3128', proxy_ssl_port='3129')) - - builder = DockerImageBuilder( - image_tag="minifi-http-proxy:latest", - dockerfile_content=dockerfile + """.format( + proxy_username="admin", + proxy_password="test101", + proxy_port="3128", + proxy_ssl_port="3129", + ) ) + + builder = DockerImageBuilder(image_tag="minifi-http-proxy:latest", dockerfile_content=dockerfile) builder.build() - super().__init__("minifi-http-proxy:latest", f"http-proxy-{test_context.scenario_id}", test_context.network) - squid_cert, squid_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + super().__init__( + "minifi-http-proxy:latest", + f"http-proxy-{test_context.scenario_id}", + test_context.network, + ) + squid_cert, squid_key = make_server_cert( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) - self.files.append(File("/etc/squid/certs/squid-cert.pem", dump_cert(squid_cert), permissions=0o666)) + self.files.append( + File( + "/etc/squid/certs/squid-cert.pem", + dump_cert(squid_cert), + permissions=0o666, + ) + ) self.files.append(File("/etc/squid/certs/squid-key.pem", dump_key(squid_key), permissions=0o666)) def deploy(self, context: MinifiTestContext | None) -> bool: @@ -65,14 +80,19 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=30, bail_condition=lambda: self.exited, - context=context + context=context, ) @retry_check(10, 1) def check_http_proxy_access(self, url): (code, output) = self.exec_run(["cat", "/var/log/squid/access.log"]) - return code == 0 and url.lower() in output.lower() \ - and ((output.count("TCP_DENIED") != 0 - and output.count("TCP_MISS") >= output.count("TCP_DENIED")) - or output.count("TCP_DENIED") == 0 and "TCP_MISS" in output - or output.count("TCP_TUNNEL") > 0) + return ( + code == 0 + and url.lower() in output.lower() + and ( + (output.count("TCP_DENIED") != 0 and output.count("TCP_MISS") >= output.count("TCP_DENIED")) + or output.count("TCP_DENIED") == 0 + and "TCP_MISS" in output + or output.count("TCP_TUNNEL") > 0 + ) + ) diff --git a/behave_framework/src/minifi_behave/containers/minifi_controller.py b/behave_framework/src/minifi_behave/containers/minifi_controller.py index 756cbc389c..fa75259c87 100644 --- a/behave_framework/src/minifi_behave/containers/minifi_controller.py +++ b/behave_framework/src/minifi_behave/containers/minifi_controller.py @@ -14,8 +14,11 @@ # limitations under the License. import logging + from minifi_behave.core.helpers import retry_check +logger = logging.getLogger(__name__) + class MinifiController: def __init__(self, minifi_container, config_path, bin_path): @@ -30,12 +33,20 @@ def set_controller_socket_properties(self): self.minifi_container.properties["controller.socket.local.any.interface"] = "false" def update_flow_config_through_controller(self): - self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--updateflow", "/tmp/resources/minifi-controller/config.yml"]) + self.minifi_container.exec_run( + [ + f"{self.bin_path}/minifi-controller", + "--updateflow", + "/tmp/resources/minifi-controller/config.yml", + ] + ) def updated_config_is_persisted(self) -> bool: exit_code, output = self.minifi_container.exec_run(["cat", self.config_path]) if exit_code != 0: - logging.error(f"Failed to read MiNiFi config file to check if updated config is persisted: {exit_code} {output}") + logger.error( + f"Failed to read MiNiFi config file to check if updated config is persisted: {exit_code} {output}" + ) return False return "2f2a3b47-f5ba-49f6-82b5-bc1c86b96f38" in output @@ -53,8 +64,8 @@ def is_component_running(self, component: str) -> bool: def get_connections(self): (_, output) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--list", "connections"]) connections = [] - for line in output.split('\n'): - if not line.startswith('[') and not line.startswith('Connection Names'): + for line in output.split("\n"): + if not line.startswith("[") and not line.startswith("Connection Names"): connections.append(line) return connections @@ -64,31 +75,31 @@ def connection_found_through_controller(self, connection: str) -> bool: def get_full_connection_count(self) -> int: (_, output) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--getfull"]) - for line in output.split('\n'): + for line in output.split("\n"): if "are full" in line: - return int(line.split(' ')[0]) + return int(line.split(" ")[0]) return -1 def get_connection_size(self, connection: str): (_, output) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--getsize", connection]) - for line in output.split('\n'): + for line in output.split("\n"): if "Size/Max of " + connection in line: - size_and_max = line.split(connection)[1].split('/') + size_and_max = line.split(connection)[1].split("/") return (int(size_and_max[0].strip()), int(size_and_max[1].strip())) return (-1, -1) def get_manifest(self) -> str: (_, output) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--manifest"]) manifest = "" - for line in output.split('\n'): - if not line.startswith('['): + for line in output.split("\n"): + if not line.startswith("["): manifest += line return manifest def create_debug_bundle(self) -> bool: (code, _) = self.minifi_container.exec_run([f"{self.bin_path}/minifi-controller", "--debug", "/tmp"]) if code != 0: - logging.error("Minifi controller debug command failed with code: %d", code) + logger.error("Minifi controller debug command failed with code: %d", code) return False (code, _) = self.minifi_container.exec_run(["test", "-f", "/tmp/debug.tar.gz"]) diff --git a/behave_framework/src/minifi_behave/containers/minifi_linux_container.py b/behave_framework/src/minifi_behave/containers/minifi_linux_container.py index d5826f4e7f..1645cbdeed 100644 --- a/behave_framework/src/minifi_behave/containers/minifi_linux_container.py +++ b/behave_framework/src/minifi_behave/containers/minifi_linux_container.py @@ -17,24 +17,31 @@ import logging import os +from pathlib import Path + from minifi_behave.containers.file import File from minifi_behave.containers.host_file import HostFile from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.ssl_utils import make_cert_without_extended_usage, make_client_cert, make_server_cert, \ - dump_cert, dump_key +from minifi_behave.core.ssl_utils import ( + dump_cert, + dump_key, + make_cert_without_extended_usage, + make_client_cert, + make_server_cert, +) from minifi_behave.minifi.minifi_flow_definition import MinifiFlowDefinition -from pathlib import Path from .container_linux import LinuxContainer from .minifi_controller import MinifiController from .minifi_protocol import MinifiProtocol +logger = logging.getLogger(__name__) CA_CERT_PATHS = [ "/usr/local/share/certs/ca-root-nss.crt", "/etc/ssl/certs/ca-certificates.crt", - "/etc/pki/tls/certs/ca-bundle.crt" + "/etc/pki/tls/certs/ca-bundle.crt", ] @@ -57,10 +64,17 @@ def __init__(self): class MinifiLinuxContainer(LinuxContainer, MinifiProtocol): - def __init__(self, container_name: str, test_context: MinifiTestContext, - deployment: NormalDeployment | FHSDeployment): - super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", - test_context.network) + def __init__( + self, + container_name: str, + test_context: MinifiTestContext, + deployment: NormalDeployment | FHSDeployment, + ): + super().__init__( + test_context.minifi_container_image, + f"{container_name}-{test_context.scenario_id}", + test_context.network, + ) self.flow_definition = MinifiFlowDefinition() self.properties: dict[str, str] = {} self.log_properties: dict[str, str] = {} @@ -68,12 +82,17 @@ def __init__(self, container_name: str, test_context: MinifiTestContext, self.deploy_timeout_seconds = 20 self.deployment_type = deployment - self.controller = MinifiController(self, f"{self.deployment_type.conf_path}/config.yml", - self.deployment_type.bin_path) - - minifi_client_cert, minifi_client_key = make_cert_without_extended_usage(common_name=self.container_name, - ca_cert=test_context.root_ca_cert, - ca_key=test_context.root_ca_key) + self.controller = MinifiController( + self, + f"{self.deployment_type.conf_path}/config.yml", + self.deployment_type.bin_path, + ) + + minifi_client_cert, minifi_client_key = make_cert_without_extended_usage( + common_name=self.container_name, + ca_cert=test_context.root_ca_cert, + ca_key=test_context.root_ca_key, + ) self.files.append(File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert))) if test_context.override_default_ca_cert_files: for ca_cert_path in CA_CERT_PATHS: @@ -81,41 +100,69 @@ def __init__(self, container_name: str, test_context: MinifiTestContext, self.files.append(File("/tmp/resources/minifi_client.crt", dump_cert(minifi_client_cert))) self.files.append(File("/tmp/resources/minifi_client.key", dump_key(minifi_client_key))) self.files.append( - File("/tmp/resources/minifi_merged_cert.crt", dump_cert(minifi_client_cert) + dump_key(minifi_client_key))) - - clientuser_cert, clientuser_key = make_client_cert("clientuser", ca_cert=test_context.root_ca_cert, - ca_key=test_context.root_ca_key) + File( + "/tmp/resources/minifi_merged_cert.crt", + dump_cert(minifi_client_cert) + dump_key(minifi_client_key), + ) + ) + + clientuser_cert, clientuser_key = make_client_cert( + "clientuser", + ca_cert=test_context.root_ca_cert, + ca_key=test_context.root_ca_key, + ) self.files.append(File("/tmp/resources/clientuser.crt", dump_cert(clientuser_cert))) self.files.append(File("/tmp/resources/clientuser.key", dump_key(clientuser_key))) - minifi_server_cert, minifi_server_key = make_server_cert(common_name=f"server-{test_context.scenario_id}", - ca_cert=test_context.root_ca_cert, - ca_key=test_context.root_ca_key) + minifi_server_cert, minifi_server_key = make_server_cert( + common_name=f"server-{test_context.scenario_id}", + ca_cert=test_context.root_ca_cert, + ca_key=test_context.root_ca_key, + ) self.files.append( - File("/tmp/resources/minifi_server.crt", dump_cert(cert=minifi_server_cert) + dump_key(minifi_server_key))) + File( + "/tmp/resources/minifi_server.crt", + dump_cert(cert=minifi_server_cert) + dump_key(minifi_server_key), + ) + ) self._fill_default_properties() self._fill_default_log_properties() def deploy(self, context: MinifiTestContext | None) -> bool: flow_config = self.flow_definition.to_yaml() - logging.info(f"Deploying MiNiFi container '{self.container_name}' with flow configuration:\n{flow_config}") + logger.info(f"Deploying MiNiFi container '{self.container_name}' with flow configuration:\n{flow_config}") self.files.append(File(f"{self.deployment_type.conf_path}/config.yml", flow_config)) self.files.append( - File(f"{self.deployment_type.conf_path}/minifi.properties", self._get_properties_file_content())) + File( + f"{self.deployment_type.conf_path}/minifi.properties", + self._get_properties_file_content(), + ) + ) self.files.append( - File(f"{self.deployment_type.conf_path}/minifi-log.properties", self._get_log_properties_file_content())) + File( + f"{self.deployment_type.conf_path}/minifi-log.properties", + self._get_log_properties_file_content(), + ) + ) resource_dir = Path(__file__).resolve().parent / "resources" / "minifi-controller" self.host_files.append( - HostFile("/tmp/resources/minifi-controller/config.yml", os.path.join(resource_dir, "config.yml"))) + HostFile( + "/tmp/resources/minifi-controller/config.yml", + os.path.join(resource_dir, "config.yml"), + ) + ) if not super().deploy(context): return False finished_str = "MiNiFi started" - return wait_for_condition(condition=lambda: finished_str in self.get_logs(), - timeout_seconds=self.deploy_timeout_seconds, bail_condition=lambda: self.exited, - context=context) + return wait_for_condition( + condition=lambda: finished_str in self.get_logs(), + timeout_seconds=self.deploy_timeout_seconds, + bail_condition=lambda: self.exited, + context=context, + ) def set_deploy_timeout_seconds(self, timeout_seconds: int): self.deploy_timeout_seconds = timeout_seconds @@ -131,7 +178,7 @@ def _fill_default_properties(self): self.properties["nifi.extension.path"] = self.deployment_type.extension_pattern self.properties["nifi.administrative.yield.duration"] = "1 sec" self.properties["nifi.bored.yield.duration"] = "100 millis" - if 'MINIFI_FIPS' in os.environ and os.environ["MINIFI_FIPS"] == "true": + if "MINIFI_FIPS" in os.environ and os.environ["MINIFI_FIPS"] == "true": self.properties["nifi.openssl.fips.support.enable"] = "true" else: self.properties["nifi.openssl.fips.support.enable"] = "false" @@ -154,5 +201,5 @@ def _get_log_properties_file_content(self): return "\n".join(lines) def add_example_python_processors(self): - run_minifi_cmd = f'{self.deployment_type.bin_path}/minifi' + run_minifi_cmd = f"{self.deployment_type.bin_path}/minifi" self.command = f'sh -c "cp -r {self.deployment_type.minifi_python_examples_path} {self.deployment_type.minifi_python_path}/examples && {run_minifi_cmd}"' diff --git a/behave_framework/src/minifi_behave/containers/minifi_protocol.py b/behave_framework/src/minifi_behave/containers/minifi_protocol.py index 1293d8a0e3..6d6067881d 100644 --- a/behave_framework/src/minifi_behave/containers/minifi_protocol.py +++ b/behave_framework/src/minifi_behave/containers/minifi_protocol.py @@ -21,14 +21,11 @@ class MinifiProtocol(Protocol): flow_definition: FlowDefinition - def set_property(self, key: str, value: str): - ... + def set_property(self, key: str, value: str): ... - def set_log_property(self, key: str, value: str): - ... + def set_log_property(self, key: str, value: str): ... - def set_deploy_timeout_seconds(self, timeout_seconds: int): - ... + def set_deploy_timeout_seconds(self, timeout_seconds: int): ... def enable_openssl_fips_mode(minifi: MinifiProtocol): @@ -42,8 +39,10 @@ def enable_log_metrics_publisher(minifi: MinifiProtocol): def conf_c2_flow_url(minifi: MinifiProtocol, scenario_id: str): - minifi.set_property("nifi.c2.flow.url", - f"http://minifi-c2-server-{scenario_id}:10090/c2/config?class=minifi-test-class") + minifi.set_property( + "nifi.c2.flow.url", + f"http://minifi-c2-server-{scenario_id}:10090/c2/config?class=minifi-test-class", + ) def set_up_ssl_properties(minifi: MinifiProtocol): diff --git a/behave_framework/src/minifi_behave/containers/minifi_win_container.py b/behave_framework/src/minifi_behave/containers/minifi_win_container.py index 020ddbe6f3..e0ba92eacd 100644 --- a/behave_framework/src/minifi_behave/containers/minifi_win_container.py +++ b/behave_framework/src/minifi_behave/containers/minifi_win_container.py @@ -14,18 +14,26 @@ # limitations under the License. +import logging +import os + +from minifi_behave.containers.directory import Directory from minifi_behave.core.minifi_test_context import MinifiTestContext from minifi_behave.minifi.minifi_flow_definition import MinifiFlowDefinition -from minifi_behave.containers.directory import Directory + from .container_windows import WindowsContainer from .minifi_protocol import MinifiProtocol -import logging -import os + +logger = logging.getLogger(__name__) class MinifiWindowsContainer(WindowsContainer, MinifiProtocol): def __init__(self, container_name: str, test_context: MinifiTestContext): - super().__init__(test_context.minifi_container_image, f"{container_name}-{test_context.scenario_id}", test_context.network) + super().__init__( + test_context.minifi_container_image, + f"{container_name}-{test_context.scenario_id}", + test_context.network, + ) self.flow_config_str: str = "" self.flow_definition = MinifiFlowDefinition() self.properties: dict[str, str] = {} @@ -35,7 +43,7 @@ def __init__(self, container_name: str, test_context: MinifiTestContext): self._fill_default_log_properties() def deploy(self, context: MinifiTestContext | None) -> bool: - logging.info(self.flow_definition.to_yaml()) + logger.info(self.flow_definition.to_yaml()) conf_dir = Directory("\\Program Files\\ApacheNiFiMiNiFi\\nifi-minifi-cpp\\conf") conf_dir.add_file("config.yml", self.flow_definition.to_yaml()) conf_dir.add_file("minifi.properties", self._get_properties_file_content()) @@ -55,7 +63,7 @@ def _fill_default_properties(self): self.properties["nifi.extension.path"] = "../extensions/*" self.properties["nifi.administrative.yield.duration"] = "1 sec" self.properties["nifi.bored.yield.duration"] = "100 millis" - if 'MINIFI_FIPS' in os.environ and os.environ["MINIFI_FIPS"] == "true": + if "MINIFI_FIPS" in os.environ and os.environ["MINIFI_FIPS"] == "true": self.properties["nifi.openssl.fips.support.enable"] = "true" else: self.properties["nifi.openssl.fips.support.enable"] = "false" diff --git a/behave_framework/src/minifi_behave/containers/nifi_container.py b/behave_framework/src/minifi_behave/containers/nifi_container.py index 8e36c4a1b6..3a7ae9fc35 100644 --- a/behave_framework/src/minifi_behave/containers/nifi_container.py +++ b/behave_framework/src/minifi_behave/containers/nifi_container.py @@ -13,82 +13,105 @@ # See the License for the specific language governing permissions and # limitations under the License. -import io import gzip +import io import logging import os from pathlib import Path -from minifi_behave.containers.file import File from minifi_behave.containers.container_linux import LinuxContainer +from minifi_behave.containers.file import File +from minifi_behave.containers.host_file import HostFile from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert from minifi_behave.minifi.nifi_flow_definition import NifiFlowDefinition -from minifi_behave.containers.host_file import HostFile -from minifi_behave.core.ssl_utils import make_server_cert -from minifi_behave.core.ssl_utils import dump_cert, dump_key +logger = logging.getLogger(__name__) class NifiContainer(LinuxContainer): - def __init__(self, test_context: MinifiTestContext, command: list[str] | None = None, use_ssl: bool = False): + def __init__( + self, + test_context: MinifiTestContext, + command: list[str] | None = None, + use_ssl: bool = False, + ): self.flow_definition = NifiFlowDefinition() name = f"nifi-{test_context.scenario_id}" if use_ssl: - entry_command = (r"/scripts/convert_cert_to_jks.sh /tmp/resources /tmp/resources/nifi_client.key /tmp/resources/nifi_client.crt /tmp/resources/root_ca.crt &&" - r"sed -i -e 's/^\(nifi.remote.input.host\)=.*/\1={name}/' " - r"-e 's/^\(nifi.remote.input.secure\)=.*/\1=true/' " - r"-e 's/^\(nifi.sensitive.props.key\)=.*/\1=secret_key_12345/' " - r"-e 's/^\(nifi.web.https.port\)=.*/\1=8443/' " - r"-e 's/^\(nifi.web.https.host\)=.*/\1={name}/' " - r"-e 's/^\(nifi.security.keystore\)=.*/\1=\/tmp\/resources\/keystore.jks/' " - r"-e 's/^\(nifi.security.keystoreType\)=.*/\1=jks/' " - r"-e 's/^\(nifi.security.keystorePasswd\)=.*/\1=passw0rd1!/' " - r"-e 's/^\(nifi.security.keyPasswd\)=.*/#\1=passw0rd1!/' " - r"-e 's/^\(nifi.security.truststore\)=.*/\1=\/tmp\/resources\/truststore.jks/' " - r"-e 's/^\(nifi.security.truststoreType\)=.*/\1=jks/' " - r"-e 's/^\(nifi.security.truststorePasswd\)=.*/\1=passw0rd1!/' " - r"-e 's/^\(nifi.remote.input.socket.port\)=.*/\1=10443/' /opt/nifi/nifi-current/conf/nifi.properties && " - r"cp /tmp/nifi_config/flow.json.gz /opt/nifi/nifi-current/conf && /opt/nifi/nifi-current/bin/nifi.sh run & " - r"nifi_pid=$! &&" - r"tail -F --pid=${{nifi_pid}} /opt/nifi/nifi-current/logs/nifi-app.log").format(name=name) + entry_command = ( + r"/scripts/convert_cert_to_jks.sh /tmp/resources /tmp/resources/nifi_client.key /tmp/resources/nifi_client.crt /tmp/resources/root_ca.crt &&" + rf"sed -i -e 's/^\(nifi.remote.input.host\)=.*/\1={name}/' " + r"-e 's/^\(nifi.remote.input.secure\)=.*/\1=true/' " + r"-e 's/^\(nifi.sensitive.props.key\)=.*/\1=secret_key_12345/' " + r"-e 's/^\(nifi.web.https.port\)=.*/\1=8443/' " + rf"-e 's/^\(nifi.web.https.host\)=.*/\1={name}/' " + r"-e 's/^\(nifi.security.keystore\)=.*/\1=\/tmp\/resources\/keystore.jks/' " + r"-e 's/^\(nifi.security.keystoreType\)=.*/\1=jks/' " + r"-e 's/^\(nifi.security.keystorePasswd\)=.*/\1=passw0rd1!/' " + r"-e 's/^\(nifi.security.keyPasswd\)=.*/#\1=passw0rd1!/' " + r"-e 's/^\(nifi.security.truststore\)=.*/\1=\/tmp\/resources\/truststore.jks/' " + r"-e 's/^\(nifi.security.truststoreType\)=.*/\1=jks/' " + r"-e 's/^\(nifi.security.truststorePasswd\)=.*/\1=passw0rd1!/' " + r"-e 's/^\(nifi.remote.input.socket.port\)=.*/\1=10443/' /opt/nifi/nifi-current/conf/nifi.properties && " + r"cp /tmp/nifi_config/flow.json.gz /opt/nifi/nifi-current/conf && /opt/nifi/nifi-current/bin/nifi.sh run & " + r"nifi_pid=$! &&" + r"tail -F --pid=${nifi_pid} /opt/nifi/nifi-current/logs/nifi-app.log" + ) else: - entry_command = (r"sed -i -e 's/^\(nifi.remote.input.host\)=.*/\1={name}/' " - r"-e 's/^\(nifi.sensitive.props.key\)=.*/\1=secret_key_12345/' " - r"-e 's/^\(nifi.remote.input.secure\)=.*/\1=false/' " - r"-e 's/^\(nifi.web.http.port\)=.*/\1=8080/' " - r"-e 's/^\(nifi.web.https.port\)=.*/\1=/' " - r"-e 's/^\(nifi.web.https.host\)=.*/\1=/' " - r"-e 's/^\(nifi.web.http.host\)=.*/\1={name}/' " - r"-e 's/^\(nifi.security.keystore\)=.*/\1=/' " - r"-e 's/^\(nifi.security.keystoreType\)=.*/\1=/' " - r"-e 's/^\(nifi.security.keystorePasswd\)=.*/\1=/' " - r"-e 's/^\(nifi.security.keyPasswd\)=.*/\1=/' " - r"-e 's/^\(nifi.security.truststore\)=.*/\1=/' " - r"-e 's/^\(nifi.security.truststoreType\)=.*/\1=/' " - r"-e 's/^\(nifi.security.truststorePasswd\)=.*/\1=/' " - r"-e 's/^\(nifi.remote.input.socket.port\)=.*/\1=10000/' /opt/nifi/nifi-current/conf/nifi.properties && " - r"cp /tmp/nifi_config/flow.json.gz /opt/nifi/nifi-current/conf && /opt/nifi/nifi-current/bin/nifi.sh run & " - r"nifi_pid=$! &&" - r"tail -F --pid=${{nifi_pid}} /opt/nifi/nifi-current/logs/nifi-app.log").format(name=name) + entry_command = ( + rf"sed -i -e 's/^\(nifi.remote.input.host\)=.*/\1={name}/' " + r"-e 's/^\(nifi.sensitive.props.key\)=.*/\1=secret_key_12345/' " + r"-e 's/^\(nifi.remote.input.secure\)=.*/\1=false/' " + r"-e 's/^\(nifi.web.http.port\)=.*/\1=8080/' " + r"-e 's/^\(nifi.web.https.port\)=.*/\1=/' " + r"-e 's/^\(nifi.web.https.host\)=.*/\1=/' " + rf"-e 's/^\(nifi.web.http.host\)=.*/\1={name}/' " + r"-e 's/^\(nifi.security.keystore\)=.*/\1=/' " + r"-e 's/^\(nifi.security.keystoreType\)=.*/\1=/' " + r"-e 's/^\(nifi.security.keystorePasswd\)=.*/\1=/' " + r"-e 's/^\(nifi.security.keyPasswd\)=.*/\1=/' " + r"-e 's/^\(nifi.security.truststore\)=.*/\1=/' " + r"-e 's/^\(nifi.security.truststoreType\)=.*/\1=/' " + r"-e 's/^\(nifi.security.truststorePasswd\)=.*/\1=/' " + r"-e 's/^\(nifi.remote.input.socket.port\)=.*/\1=10000/' /opt/nifi/nifi-current/conf/nifi.properties && " + r"cp /tmp/nifi_config/flow.json.gz /opt/nifi/nifi-current/conf && /opt/nifi/nifi-current/bin/nifi.sh run & " + r"nifi_pid=$! &&" + r"tail -F --pid=${nifi_pid} /opt/nifi/nifi-current/logs/nifi-app.log" + ) if not command: command = ["/bin/sh", "-c", entry_command] - super().__init__("apache/nifi:" + NifiFlowDefinition.NIFI_VERSION, name, test_context.network, entrypoint=command) + super().__init__( + "apache/nifi:" + NifiFlowDefinition.NIFI_VERSION, + name, + test_context.network, + entrypoint=command, + ) resource_dir = Path(__file__).resolve().parent / "resources" / "nifi" - self.host_files.append(HostFile("/scripts/convert_cert_to_jks.sh", os.path.join(resource_dir, "convert_cert_to_jks.sh"))) + self.host_files.append( + HostFile( + "/scripts/convert_cert_to_jks.sh", + os.path.join(resource_dir, "convert_cert_to_jks.sh"), + ) + ) - nifi_client_cert, nifi_client_key = make_server_cert(common_name=f"nifi-{test_context.scenario_id}", ca_cert=test_context.root_ca_cert, ca_key=test_context.root_ca_key) + nifi_client_cert, nifi_client_key = make_server_cert( + common_name=f"nifi-{test_context.scenario_id}", + ca_cert=test_context.root_ca_cert, + ca_key=test_context.root_ca_key, + ) self.files.append(File("/tmp/resources/root_ca.crt", dump_cert(test_context.root_ca_cert))) self.files.append(File("/tmp/resources/nifi_client.crt", dump_cert(nifi_client_cert))) self.files.append(File("/tmp/resources/nifi_client.key", dump_key(nifi_client_key))) def deploy(self, context: MinifiTestContext | None) -> bool: flow_config = self.flow_definition.to_json() - logging.info(f"Deploying NiFi container '{self.container_name}' with flow configuration:\n{flow_config}") + logger.info(f"Deploying NiFi container '{self.container_name}' with flow configuration:\n{flow_config}") buffer = io.BytesIO() - with gzip.GzipFile(fileobj=buffer, mode='wb') as gz_file: + with gzip.GzipFile(fileobj=buffer, mode="wb") as gz_file: gz_file.write(flow_config.encode()) gzipped_bytes = buffer.getvalue() @@ -100,4 +123,5 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=300, bail_condition=lambda: self.exited, - context=context) + context=context, + ) diff --git a/behave_framework/src/minifi_behave/core/helpers.py b/behave_framework/src/minifi_behave/core/helpers.py index 128aa4e017..f35b34c111 100644 --- a/behave_framework/src/minifi_behave/core/helpers.py +++ b/behave_framework/src/minifi_behave/core/helpers.py @@ -17,14 +17,17 @@ from __future__ import annotations +import functools import logging import time -import functools from collections.abc import Callable -import docker from minifi_behave.core.minifi_test_context import MinifiTestContext +import docker + +logger = logging.getLogger(__name__) + def log_due_to_failure(context: MinifiTestContext | None): if context is not None: @@ -32,19 +35,25 @@ def log_due_to_failure(context: MinifiTestContext | None): container.log_app_output() -def check_condition_after_wait(condition: Callable[[], bool], context: MinifiTestContext | None, wait_time: int) -> bool: +def check_condition_after_wait( + condition: Callable[[], bool], context: MinifiTestContext | None, wait_time: int +) -> bool: time.sleep(wait_time) if not condition(): - logging.warning("Condition not met after wait") + logger.warning("Condition not met after wait") log_due_to_failure(context) return False return True -def wait_for_condition(condition: Callable[[], bool], timeout_seconds: float, bail_condition: Callable[[], bool], - context: MinifiTestContext | None) -> bool: +def wait_for_condition( + condition: Callable[[], bool], + timeout_seconds: float, + bail_condition: Callable[[], bool], + context: MinifiTestContext | None, +) -> bool: if bail_condition(): - logging.warning("Bail condition evaluated to 'True', aborting wait.") + logger.warning("Bail condition evaluated to 'True', aborting wait.") log_due_to_failure(context) return False start_time = time.monotonic() @@ -53,7 +62,7 @@ def wait_for_condition(condition: Callable[[], bool], timeout_seconds: float, ba if condition(): return True if bail_condition(): - logging.warning("Bail condition evaluated to 'True', aborting wait.") + logger.warning("Bail condition evaluated to 'True', aborting wait.") log_due_to_failure(context) return False remaining_time = timeout_seconds - (time.monotonic() - start_time) @@ -61,23 +70,25 @@ def wait_for_condition(condition: Callable[[], bool], timeout_seconds: float, ba if sleep_time > 0: time.sleep(sleep_time) except Exception as ex: - logging.warning("Exception while waiting for condition: %s", ex) + logger.warning("Exception while waiting for condition: %s", ex) log_due_to_failure(context) return False - logging.warning("Timed out after %d seconds while waiting for condition", timeout_seconds) + logger.warning(f"Timed out after {timeout_seconds} seconds while waiting for condition") log_due_to_failure(context) return False def run_cmd_in_docker_image(image_name: str, cmd: str | list, network: str) -> str: client = docker.from_env() - output = client.containers.run(image=image_name, - command=cmd, - remove=True, - stdout=True, - stderr=True, - network=network, - detach=False) + output = client.containers.run( + image=image_name, + command=cmd, + remove=True, + stdout=True, + stderr=True, + network=network, + detach=False, + ) return output.decode("utf-8") @@ -90,6 +101,7 @@ def retry_check(max_tries: int = 5, retry_interval_seconds: int = 1): Decorator for retrying a checker function that returns a boolean. The decorated function is called repeatedly until it returns True or the maximum number of attempts is reached. The maximum number of attempts and the interval between attempts in seconds can be configured. """ + def retry_check_func(func): @functools.wraps(func) def retry_wrapper(*args, **kwargs): @@ -99,5 +111,7 @@ def retry_wrapper(*args, **kwargs): if i < max_tries - 1: time.sleep(retry_interval_seconds) return False + return retry_wrapper + return retry_check_func diff --git a/behave_framework/src/minifi_behave/core/hooks.py b/behave_framework/src/minifi_behave/core/hooks.py index bec809f1ce..862ab13100 100644 --- a/behave_framework/src/minifi_behave/core/hooks.py +++ b/behave_framework/src/minifi_behave/core/hooks.py @@ -14,24 +14,26 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import os -import docker import types -import logging from pathlib import Path from behave.model import Scenario from behave.runner import Context - -from minifi_behave.core.ssl_utils import make_self_signed_cert from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import make_self_signed_cert + +import docker + +logger = logging.getLogger(__name__) def get_minifi_container_image(): - if 'MINIFI_TAG_PREFIX' in os.environ and 'MINIFI_VERSION' in os.environ: - minifi_tag_prefix = os.environ['MINIFI_TAG_PREFIX'] - minifi_version = os.environ['MINIFI_VERSION'] - return 'apacheminificpp:' + minifi_tag_prefix + minifi_version + if "MINIFI_TAG_PREFIX" in os.environ and "MINIFI_VERSION" in os.environ: + minifi_tag_prefix = os.environ["MINIFI_TAG_PREFIX"] + minifi_version = os.environ["MINIFI_VERSION"] + return "apacheminificpp:" + minifi_tag_prefix + minifi_version return "apacheminificpp:behave" @@ -40,13 +42,16 @@ def inject_scenario_id(context: MinifiTestContext, step): step.name = step.name.replace("${scenario_id}", context.scenario_id) if getattr(step, "table", None): for row in step.table: - row.cells = [cell.replace("${scenario_id}", context.scenario_id) if "${scenario_id}" in cell else cell for cell in row.cells] + row.cells = [ + cell.replace("${scenario_id}", context.scenario_id) if "${scenario_id}" in cell else cell + for cell in row.cells + ] if hasattr(step, "text") and step.text and "${scenario_id}" in step.text: step.text = step.text.replace("${scenario_id}", context.scenario_id) def common_before_scenario(context: Context, scenario: Scenario): - if "SUPPORTS_WINDOWS" not in scenario.effective_tags and os.name == 'nt': + if "SUPPORTS_WINDOWS" not in scenario.effective_tags and os.name == "nt": scenario.skip("No windows support") return @@ -55,21 +60,22 @@ def common_before_scenario(context: Context, scenario: Scenario): method_map = { "get_or_create_minifi_container": MinifiTestContext.get_or_create_minifi_container, - "get_or_create_default_minifi_container": MinifiTestContext.get_or_create_default_minifi_container + "get_or_create_default_minifi_container": MinifiTestContext.get_or_create_default_minifi_container, } for attr, method in method_map.items(): if not hasattr(context, attr): setattr(context, attr, types.MethodType(method, context)) - logging.info("Running scenario: %s", scenario) - context.scenario_id = scenario.filename.rsplit("/", 1)[1].split(".")[0] + "-" + str( - scenario.parent.scenarios.index(scenario)) + logger.info("Running scenario: %s", scenario) + context.scenario_id = ( + scenario.filename.rsplit("/", 1)[1].split(".")[0] + "-" + str(scenario.parent.scenarios.index(scenario)) + ) network_name = f"{context.scenario_id}-net" docker_client = docker.client.from_env() try: existing_network = docker_client.networks.get(network_name) - logging.warning(f"Found existing network '{network_name}'. Removing it first.") + logger.warning(f"Found existing network '{network_name}'. Removing it first.") existing_network.remove() except docker.errors.NotFound: pass # No existing network found, which is good. @@ -98,8 +104,8 @@ def common_after_scenario(context: MinifiTestContext, scenario: Scenario): with open(scenario_info_path, "w") as f: f.write(header) - if hasattr(context, 'containers'): + if hasattr(context, "containers"): for container in context.containers.values(): container.clean_up() - if hasattr(context, 'network'): + if hasattr(context, "network"): context.network.remove() diff --git a/behave_framework/src/minifi_behave/core/minifi_test_context.py b/behave_framework/src/minifi_behave/core/minifi_test_context.py index 59a1e04f2c..9106272218 100644 --- a/behave_framework/src/minifi_behave/core/minifi_test_context.py +++ b/behave_framework/src/minifi_behave/core/minifi_test_context.py @@ -18,15 +18,15 @@ from __future__ import annotations import os -import docker + from behave.runner import Context from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey from cryptography.x509 import Certificate from docker.models.networks import Network - from minifi_behave.containers.container_protocol import ContainerProtocol from minifi_behave.containers.minifi_protocol import MinifiProtocol +import docker DEFAULT_MINIFI_CONTAINER_NAME = "minifi-primary" @@ -47,14 +47,27 @@ class MinifiTestContext(Context): def get_or_create_minifi_container(self, container_name: str) -> MinifiContainer: if container_name not in self.containers: - if os.name == 'nt': - from minifi_behave.containers.minifi_win_container import MinifiWindowsContainer + if os.name == "nt": + from minifi_behave.containers.minifi_win_container import ( + MinifiWindowsContainer, + ) + minifi_container = MinifiWindowsContainer(container_name, self) - elif 'MINIFI_INSTALLATION_TYPE=FHS' in str(docker.from_env().images.get(self.minifi_container_image).history()): - from minifi_behave.containers.minifi_linux_container import MinifiLinuxContainer, FHSDeployment + elif "MINIFI_INSTALLATION_TYPE=FHS" in str( + docker.from_env().images.get(self.minifi_container_image).history() + ): + from minifi_behave.containers.minifi_linux_container import ( + FHSDeployment, + MinifiLinuxContainer, + ) + minifi_container = MinifiLinuxContainer(container_name, self, FHSDeployment()) else: - from minifi_behave.containers.minifi_linux_container import MinifiLinuxContainer, NormalDeployment + from minifi_behave.containers.minifi_linux_container import ( + MinifiLinuxContainer, + NormalDeployment, + ) + minifi_container = MinifiLinuxContainer(container_name, self, NormalDeployment()) self.containers[container_name] = minifi_container return self.containers[container_name] diff --git a/behave_framework/src/minifi_behave/core/ssl_utils.py b/behave_framework/src/minifi_behave/core/ssl_utils.py index 3fc3c1de52..4b2cae9b34 100644 --- a/behave_framework/src/minifi_behave/core/ssl_utils.py +++ b/behave_framework/src/minifi_behave/core/ssl_utils.py @@ -16,12 +16,11 @@ import datetime from cryptography import x509 -from cryptography.hazmat.primitives import hashes -from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives import hashes, serialization from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.asymmetric.rsa import RSAPrivateKey from cryptography.x509 import Certificate, ExtendedKeyUsage -from cryptography.x509.oid import NameOID, ExtendedKeyUsageOID +from cryptography.x509.oid import ExtendedKeyUsageOID, NameOID def gen_cert() -> tuple[Certificate, RSAPrivateKey]: @@ -116,7 +115,9 @@ def _make_cert( return cert, key -def make_client_cert(common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey) -> tuple[Certificate, RSAPrivateKey]: +def make_client_cert( + common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey +) -> tuple[Certificate, RSAPrivateKey]: return _make_cert( common_name, ca_cert, @@ -125,7 +126,9 @@ def make_client_cert(common_name: str, ca_cert: Certificate, ca_key: RSAPrivateK ) -def make_server_cert(common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey) -> tuple[Certificate, RSAPrivateKey]: +def make_server_cert( + common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey +) -> tuple[Certificate, RSAPrivateKey]: return _make_cert( common_name, ca_cert, @@ -134,7 +137,9 @@ def make_server_cert(common_name: str, ca_cert: Certificate, ca_key: RSAPrivateK ) -def make_cert_without_extended_usage(common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey) -> tuple[Certificate, RSAPrivateKey]: +def make_cert_without_extended_usage( + common_name: str, ca_cert: Certificate, ca_key: RSAPrivateKey +) -> tuple[Certificate, RSAPrivateKey]: return _make_cert(common_name, ca_cert, ca_key, None) diff --git a/behave_framework/src/minifi_behave/minifi/controller_service.py b/behave_framework/src/minifi_behave/minifi/controller_service.py index 58a9136ae2..2af194e1e3 100644 --- a/behave_framework/src/minifi_behave/minifi/controller_service.py +++ b/behave_framework/src/minifi_behave/minifi/controller_service.py @@ -30,6 +30,11 @@ def add_property(self, property_name: str, property_value: str): self.properties[property_name] = property_value def to_yaml_dict(self) -> dict: - data = {'name': self.name, 'id': self.id, 'class': self.class_name, 'Properties': self.properties} + data = { + "name": self.name, + "id": self.id, + "class": self.class_name, + "Properties": self.properties, + } return data diff --git a/behave_framework/src/minifi_behave/minifi/flow_definition.py b/behave_framework/src/minifi_behave/minifi/flow_definition.py index a636cf353a..acc5a9a09e 100644 --- a/behave_framework/src/minifi_behave/minifi/flow_definition.py +++ b/behave_framework/src/minifi_behave/minifi/flow_definition.py @@ -20,11 +20,11 @@ from .connection import Connection from .controller_service import ControllerService from .funnel import Funnel +from .input_port import InputPort +from .output_port import OutputPort from .parameter_context import ParameterContext from .processor import Processor from .remote_process_group import RemoteProcessGroup -from .input_port import InputPort -from .output_port import OutputPort class FlowDefinition(ABC): @@ -92,11 +92,20 @@ def get_processor(self, processor_name: str) -> Processor | None: return next((proc for proc in self.processors if proc.name == processor_name), None) def get_controller_service(self, controller_service_name: str) -> ControllerService | None: - return next((controller for controller in self.controller_services if controller.name == controller_service_name), None) + return next( + (controller for controller in self.controller_services if controller.name == controller_service_name), + None, + ) def get_parameter_context(self, parameter_context_name: str) -> ParameterContext | None: - return next((parameter_context for parameter_context in self.parameter_contexts if - parameter_context.name == parameter_context_name), None) + return next( + ( + parameter_context + for parameter_context in self.parameter_contexts + if parameter_context.name == parameter_context_name + ), + None, + ) def get_remote_process_group(self, rpg_name: str) -> RemoteProcessGroup | None: return next((rpg for rpg in self.remote_process_groups if rpg.name == rpg_name), None) diff --git a/behave_framework/src/minifi_behave/minifi/funnel.py b/behave_framework/src/minifi_behave/minifi/funnel.py index 998b453375..c38322e5a9 100644 --- a/behave_framework/src/minifi_behave/minifi/funnel.py +++ b/behave_framework/src/minifi_behave/minifi/funnel.py @@ -28,4 +28,7 @@ def __repr__(self): def to_yaml_dict(self) -> dict: # Funnels have a simpler representation in the MiNiFi YAML - return {'id': self.id, 'name': self.name, } + return { + "id": self.id, + "name": self.name, + } diff --git a/behave_framework/src/minifi_behave/minifi/input_port.py b/behave_framework/src/minifi_behave/minifi/input_port.py index 1af6d20972..3f7e51a5f1 100644 --- a/behave_framework/src/minifi_behave/minifi/input_port.py +++ b/behave_framework/src/minifi_behave/minifi/input_port.py @@ -15,6 +15,7 @@ # limitations under the License. # + class InputPort: def __init__(self, name: str, id: str): self.id: str = id @@ -26,16 +27,13 @@ def to_json_dict(self): "instanceIdentifier": self.id, "name": self.name, "comments": "", - "position": { - "x": 0, - "y": 0 - }, + "position": {"x": 0, "y": 0}, "type": "INPUT_PORT", "concurrentlySchedulableTaskCount": 1, "scheduledState": "RUNNING", "allowRemoteAccess": True, "portFunction": "STANDARD", "componentType": "INPUT_PORT", - "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8" + "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8", } return data diff --git a/behave_framework/src/minifi_behave/minifi/minifi_flow_definition.py b/behave_framework/src/minifi_behave/minifi/minifi_flow_definition.py index e01a91880a..2874e9fe5b 100644 --- a/behave_framework/src/minifi_behave/minifi/minifi_flow_definition.py +++ b/behave_framework/src/minifi_behave/minifi/minifi_flow_definition.py @@ -32,22 +32,33 @@ def to_yaml(self) -> str: processors_by_name = {p.name: p for p in self.processors} funnels_by_name = {f.name: f for f in self.funnels} remote_input_ports_by_name = {port.name: port for rpg in self.remote_process_groups for port in rpg.input_ports} - remote_output_ports_by_name = {port.name: port for rpg in self.remote_process_groups for port in rpg.output_ports} + remote_output_ports_by_name = { + port.name: port for rpg in self.remote_process_groups for port in rpg.output_ports + } - connectables_by_name = {**processors_by_name, **funnels_by_name, **remote_input_ports_by_name, **remote_output_ports_by_name} + connectables_by_name = { + **processors_by_name, + **funnels_by_name, + **remote_input_ports_by_name, + **remote_output_ports_by_name, + } if len(self.parameter_contexts) > 0: parameter_context_name = self.parameter_contexts[0].name else: - parameter_context_name = '' + parameter_context_name = "" # Build the final dictionary structure - config = {'MiNiFi Config Version': 3, 'Flow Controller': {'name': self.flow_name}, - 'Parameter Contexts': [p.to_yaml_dict() for p in self.parameter_contexts], - 'Processors': [p.to_yaml_dict() for p in self.processors], - 'Funnels': [f.to_yaml_dict() for f in self.funnels], 'Connections': [], - 'Controller Services': [c.to_yaml_dict() for c in self.controller_services], - 'Remote Processing Groups': [rpg.to_yaml_dict() for rpg in self.remote_process_groups], - 'Parameter Context Name': parameter_context_name} + config = { + "MiNiFi Config Version": 3, + "Flow Controller": {"name": self.flow_name}, + "Parameter Contexts": [p.to_yaml_dict() for p in self.parameter_contexts], + "Processors": [p.to_yaml_dict() for p in self.processors], + "Funnels": [f.to_yaml_dict() for f in self.funnels], + "Connections": [], + "Controller Services": [c.to_yaml_dict() for c in self.controller_services], + "Remote Processing Groups": [rpg.to_yaml_dict() for rpg in self.remote_process_groups], + "Parameter Context Name": parameter_context_name, + } # Build the connections list by looking up processor IDs for conn in self.connections: @@ -56,11 +67,18 @@ def to_yaml(self) -> str: if not source_proc or not dest_proc: raise ValueError( - f"Could not find processors for connection from '{conn.source_name}' to '{conn.target_name}'") + f"Could not find processors for connection from '{conn.source_name}' to '{conn.target_name}'" + ) - config['Connections'].append( - {'name': f"{conn.source_name}/{conn.source_relationship}/{conn.target_name}", 'id': conn.id, - 'source id': source_proc.id, 'source relationship name': conn.source_relationship, - 'destination id': dest_proc.id, "drop empty": conn.drop_empty_flowfiles}) + config["Connections"].append( + { + "name": f"{conn.source_name}/{conn.source_relationship}/{conn.target_name}", + "id": conn.id, + "source id": source_proc.id, + "source relationship name": conn.source_relationship, + "destination id": dest_proc.id, + "drop empty": conn.drop_empty_flowfiles, + } + ) return yaml.dump(config, sort_keys=False, indent=2, width=120) diff --git a/behave_framework/src/minifi_behave/minifi/nifi_flow_definition.py b/behave_framework/src/minifi_behave/minifi/nifi_flow_definition.py index a41e248bcf..02459b3b18 100644 --- a/behave_framework/src/minifi_behave/minifi/nifi_flow_definition.py +++ b/behave_framework/src/minifi_behave/minifi/nifi_flow_definition.py @@ -17,21 +17,19 @@ import json import uuid + from .flow_definition import FlowDefinition class NifiFlowDefinition(FlowDefinition): - NIFI_VERSION: str = '2.7.2' + NIFI_VERSION: str = "2.7.2" def __init__(self, flow_name: str = "NiFi Flow"): super().__init__(flow_name) def to_json(self) -> str: config = { - "encodingVersion": { - "majorVersion": 2, - "minorVersion": 0 - }, + "encodingVersion": {"majorVersion": 2, "minorVersion": 0}, "maxTimerDrivenThreadCount": 10, "maxEventDrivenThreadCount": 1, "registries": [], @@ -45,10 +43,7 @@ def to_json(self) -> str: "instanceIdentifier": str(uuid.uuid4()), "name": "NiFi Flow", "comments": "", - "position": { - "x": 0, - "y": 0 - }, + "position": {"x": 0, "y": 0}, "processGroups": [], "remoteProcessGroups": [], "processors": [], @@ -67,49 +62,52 @@ def to_json(self) -> str: "statelessFlowTimeout": "1 min", "flowFileConcurrency": "UNBOUNDED", "flowFileOutboundPolicy": "STREAM_WHEN_AVAILABLE", - "componentType": "PROCESS_GROUP" - } + "componentType": "PROCESS_GROUP", + }, } processors_by_name = {p.name: p for p in self.processors} processors_node = config["rootGroup"]["processors"] for proc in self.processors: - processors_node.append({ - "identifier": str(proc.id), - "instanceIdentifier": str(proc.id), - "name": proc.name, - "comments": "", - "position": { - "x": 0, - "y": 0 - }, - "type": 'org.apache.nifi.processors.standard.' + proc.class_name, - "bundle": { - "group": "org.apache.nifi", - "artifact": "nifi-standard-nar", - "version": self.NIFI_VERSION - }, - "properties": {key: value for key, value in proc.properties.items() if key}, - "propertyDescriptors": {}, - "style": {}, - "schedulingPeriod": "0 sec" if proc.scheduling_strategy == "EVENT_DRIVEN" else proc.scheduling_period, - "schedulingStrategy": "TIMER_DRIVEN", - "executionNode": "ALL", - "penaltyDuration": "5 sec", - "yieldDuration": "1 sec", - "bulletinLevel": "WARN", - "runDurationMillis": "0", - "concurrentlySchedulableTaskCount": proc.max_concurrent_tasks if proc.max_concurrent_tasks is not None else 1, - "autoTerminatedRelationships": proc.auto_terminated_relationships, - "scheduledState": "RUNNING", - "retryCount": 10, - "retriedRelationships": [], - "backoffMechanism": "PENALIZE_FLOWFILE", - "maxBackoffPeriod": "10 mins", - "componentType": "PROCESSOR", - "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8" - }) + processors_node.append( + { + "identifier": str(proc.id), + "instanceIdentifier": str(proc.id), + "name": proc.name, + "comments": "", + "position": {"x": 0, "y": 0}, + "type": "org.apache.nifi.processors.standard." + proc.class_name, + "bundle": { + "group": "org.apache.nifi", + "artifact": "nifi-standard-nar", + "version": self.NIFI_VERSION, + }, + "properties": {key: value for key, value in proc.properties.items() if key}, + "propertyDescriptors": {}, + "style": {}, + "schedulingPeriod": "0 sec" + if proc.scheduling_strategy == "EVENT_DRIVEN" + else proc.scheduling_period, + "schedulingStrategy": "TIMER_DRIVEN", + "executionNode": "ALL", + "penaltyDuration": "5 sec", + "yieldDuration": "1 sec", + "bulletinLevel": "WARN", + "runDurationMillis": "0", + "concurrentlySchedulableTaskCount": proc.max_concurrent_tasks + if proc.max_concurrent_tasks is not None + else 1, + "autoTerminatedRelationships": proc.auto_terminated_relationships, + "scheduledState": "RUNNING", + "retryCount": 10, + "retriedRelationships": [], + "backoffMechanism": "PENALIZE_FLOWFILE", + "maxBackoffPeriod": "10 mins", + "componentType": "PROCESSOR", + "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8", + } + ) connections_node = config["rootGroup"]["connections"] @@ -131,42 +129,45 @@ def to_json(self) -> str: dest_type = "OUTPUT_PORT" if not source_proc or not dest_proc: raise ValueError( - f"Could not find processors for connection from '{conn.source_name}' to '{conn.target_name}'") + f"Could not find processors for connection from '{conn.source_name}' to '{conn.target_name}'" + ) - connections_node.append({ - "identifier": conn.id, - "instanceIdentifier": conn.id, - "name": f"{conn.source_name}/{conn.source_relationship}/{conn.target_name}", - "source": { - "id": source_proc.id, - "type": source_type, - "groupId": "9802c873-3322-3b60-a71d-732d02bd60f8", - "name": conn.source_name, - "comments": "", - "instanceIdentifier": source_proc.id - }, - "destination": { - "id": dest_proc.id, - "type": dest_type, - "groupId": "9802c873-3322-3b60-a71d-732d02bd60f8", - "name": dest_proc.name, - "comments": "", - "instanceIdentifier": dest_proc.id - }, - "labelIndex": 1, - "zIndex": 0, - "selectedRelationships": [conn.source_relationship], - "backPressureObjectThreshold": 10, - "backPressureDataSizeThreshold": "50 B", - "flowFileExpiration": "0 sec", - "prioritizers": [], - "bends": [], - "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", - "partitioningAttribute": "", - "loadBalanceCompression": "DO_NOT_COMPRESS", - "componentType": "CONNECTION", - "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8" - }) + connections_node.append( + { + "identifier": conn.id, + "instanceIdentifier": conn.id, + "name": f"{conn.source_name}/{conn.source_relationship}/{conn.target_name}", + "source": { + "id": source_proc.id, + "type": source_type, + "groupId": "9802c873-3322-3b60-a71d-732d02bd60f8", + "name": conn.source_name, + "comments": "", + "instanceIdentifier": source_proc.id, + }, + "destination": { + "id": dest_proc.id, + "type": dest_type, + "groupId": "9802c873-3322-3b60-a71d-732d02bd60f8", + "name": dest_proc.name, + "comments": "", + "instanceIdentifier": dest_proc.id, + }, + "labelIndex": 1, + "zIndex": 0, + "selectedRelationships": [conn.source_relationship], + "backPressureObjectThreshold": 10, + "backPressureDataSizeThreshold": "50 B", + "flowFileExpiration": "0 sec", + "prioritizers": [], + "bends": [], + "loadBalanceStrategy": "DO_NOT_LOAD_BALANCE", + "partitioningAttribute": "", + "loadBalanceCompression": "DO_NOT_COMPRESS", + "componentType": "CONNECTION", + "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8", + } + ) config["rootGroup"]["inputPorts"] = [input_port.to_json_dict() for input_port in self.input_ports] config["rootGroup"]["outputPorts"] = [output_port.to_json_dict() for output_port in self.output_ports] diff --git a/behave_framework/src/minifi_behave/minifi/output_port.py b/behave_framework/src/minifi_behave/minifi/output_port.py index ca2ce57272..e3aa174a07 100644 --- a/behave_framework/src/minifi_behave/minifi/output_port.py +++ b/behave_framework/src/minifi_behave/minifi/output_port.py @@ -15,6 +15,7 @@ # limitations under the License. # + class OutputPort: def __init__(self, name: str, id: str): self.id: str = id @@ -26,16 +27,13 @@ def to_json_dict(self): "instanceIdentifier": self.id, "name": self.name, "comments": "", - "position": { - "x": 0, - "y": 0 - }, + "position": {"x": 0, "y": 0}, "type": "OUTPUT_PORT", "concurrentlySchedulableTaskCount": 1, "scheduledState": "RUNNING", "allowRemoteAccess": True, "portFunction": "STANDARD", "componentType": "OUTPUT_PORT", - "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8" + "groupIdentifier": "9802c873-3322-3b60-a71d-732d02bd60f8", } return data diff --git a/behave_framework/src/minifi_behave/minifi/parameter.py b/behave_framework/src/minifi_behave/minifi/parameter.py index 5f681430a7..e55b30372f 100644 --- a/behave_framework/src/minifi_behave/minifi/parameter.py +++ b/behave_framework/src/minifi_behave/minifi/parameter.py @@ -15,6 +15,7 @@ # limitations under the License. # + class Parameter: def __init__(self, name: str, value: str, is_sensitive: bool): self.name: str = name @@ -23,6 +24,10 @@ def __init__(self, name: str, value: str, is_sensitive: bool): self.is_sensitive: bool = is_sensitive def to_yaml_dict(self): - data = {'name': self.name, 'value': self.value, 'description': self.description, - 'sensitive': self.is_sensitive, } + data = { + "name": self.name, + "value": self.value, + "description": self.description, + "sensitive": self.is_sensitive, + } return data diff --git a/behave_framework/src/minifi_behave/minifi/parameter_context.py b/behave_framework/src/minifi_behave/minifi/parameter_context.py index 58cc9161eb..55c6bad9ba 100644 --- a/behave_framework/src/minifi_behave/minifi/parameter_context.py +++ b/behave_framework/src/minifi_behave/minifi/parameter_context.py @@ -28,7 +28,7 @@ def __init__(self, name: str): def to_yaml_dict(self) -> dict: return { - 'id': self.id, - 'name': self.name, - 'Parameters': [p.to_yaml_dict() for p in self.parameters], + "id": self.id, + "name": self.name, + "Parameters": [p.to_yaml_dict() for p in self.parameters], } diff --git a/behave_framework/src/minifi_behave/minifi/processor.py b/behave_framework/src/minifi_behave/minifi/processor.py index 193b8deaf4..c907bcfe96 100644 --- a/behave_framework/src/minifi_behave/minifi/processor.py +++ b/behave_framework/src/minifi_behave/minifi/processor.py @@ -19,8 +19,14 @@ class Processor: - def __init__(self, class_name: str, proc_name: str, scheduling_strategy: str = "TIMER_DRIVEN", - scheduling_period: str = "1 sec", penalization_period: str = "1 sec"): + def __init__( + self, + class_name: str, + proc_name: str, + scheduling_strategy: str = "TIMER_DRIVEN", + scheduling_period: str = "1 sec", + penalization_period: str = "1 sec", + ): self.class_name = class_name self.id: str = str(uuid.uuid4()) self.name = proc_name @@ -46,18 +52,18 @@ def __repr__(self): def to_yaml_dict(self) -> dict: data = { - 'name': self.name, - 'id': self.id, - 'class': self.class_name, - 'scheduling strategy': self.scheduling_strategy, - 'scheduling period': self.scheduling_period, - 'penalization period': self.penalization_period, + "name": self.name, + "id": self.id, + "class": self.class_name, + "scheduling strategy": self.scheduling_strategy, + "scheduling period": self.scheduling_period, + "penalization period": self.penalization_period, } if self.auto_terminated_relationships: - data['auto-terminated relationships list'] = self.auto_terminated_relationships + data["auto-terminated relationships list"] = self.auto_terminated_relationships if self.max_concurrent_tasks is not None: - data['max concurrent tasks'] = self.max_concurrent_tasks + data["max concurrent tasks"] = self.max_concurrent_tasks # The YAML format capitalizes 'Properties' - data['Properties'] = self.properties + data["Properties"] = self.properties return data diff --git a/behave_framework/src/minifi_behave/minifi/remote_port.py b/behave_framework/src/minifi_behave/minifi/remote_port.py index 5e0d7ef677..7f456a6ba7 100644 --- a/behave_framework/src/minifi_behave/minifi/remote_port.py +++ b/behave_framework/src/minifi_behave/minifi/remote_port.py @@ -28,6 +28,11 @@ def add_property(self, property_name: str, property_value: str): self.properties[property_name] = property_value def to_yaml_dict(self): - data = {'id': self.id, 'name': self.name, 'use compression': self.use_compression, 'max concurrent tasks': 1, - 'Properties': self.properties} + data = { + "id": self.id, + "name": self.name, + "use compression": self.use_compression, + "max concurrent tasks": 1, + "Properties": self.properties, + } return data diff --git a/behave_framework/src/minifi_behave/minifi/remote_process_group.py b/behave_framework/src/minifi_behave/minifi/remote_process_group.py index ae8b697535..0298264bfc 100644 --- a/behave_framework/src/minifi_behave/minifi/remote_process_group.py +++ b/behave_framework/src/minifi_behave/minifi/remote_process_group.py @@ -15,6 +15,7 @@ # limitations under the License. # import uuid + from .remote_port import RemotePort @@ -38,7 +39,14 @@ def get_input_port(self, port_name: str) -> RemotePort | None: return next((port for port in self.input_ports if port.name == port_name), None) def to_yaml_dict(self): - data = {'id': self.id, 'name': self.name, 'timeout': '30 sec', 'transport protocol': self.protocol, - 'url': self.address, 'yield period': '3 sec', 'Input Ports': [port.to_yaml_dict() for port in self.input_ports], - 'Output Ports': [port.to_yaml_dict() for port in self.output_ports]} + data = { + "id": self.id, + "name": self.name, + "timeout": "30 sec", + "transport protocol": self.protocol, + "url": self.address, + "yield period": "3 sec", + "Input Ports": [port.to_yaml_dict() for port in self.input_ports], + "Output Ports": [port.to_yaml_dict() for port in self.output_ports], + } return data diff --git a/behave_framework/src/minifi_behave/steps/checking_steps.py b/behave_framework/src/minifi_behave/steps/checking_steps.py index 418e875c93..83ff073b7d 100644 --- a/behave_framework/src/minifi_behave/steps/checking_steps.py +++ b/behave_framework/src/minifi_behave/steps/checking_steps.py @@ -14,15 +14,21 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import time import re +import time import humanfriendly -from behave import then, step - +from behave import step, then from minifi_behave.containers.http_proxy_container import HttpProxy -from minifi_behave.core.helpers import wait_for_condition, check_condition_after_wait, log_due_to_failure -from minifi_behave.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext +from minifi_behave.core.helpers import ( + check_condition_after_wait, + log_due_to_failure, + wait_for_condition, +) +from minifi_behave.core.minifi_test_context import ( + DEFAULT_MINIFI_CONTAINER_NAME, + MinifiTestContext, +) @then('a file with the content "{content}" is placed on the path "{path}" in less than {duration}') @@ -31,41 +37,76 @@ def verify_file_content_on_path(context: MinifiTestContext, content: str, path: timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].path_with_content_exists(path, new_content), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, context=context) - - -@then('in the "{container_name}" container a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') -def verify_single_file_content_in_container_directory(context: MinifiTestContext, container_name: str, content: str, directory: str, duration: str): + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) + + +@then( + 'in the "{container_name}" container a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}' +) +def verify_single_file_content_in_container_directory( + context: MinifiTestContext, + container_name: str, + content: str, + directory: str, + duration: str, +): new_content = content.replace("\\n", "\n") timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.containers[container_name].directory_has_single_file_with_content(directory, new_content), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[container_name].exited, context=context) + condition=lambda: context.containers[container_name].directory_has_single_file_with_content( + directory, new_content + ), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[container_name].exited, + context=context, + ) @then('a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') @then("a single file with the content '{content}' is placed in the '{directory}' directory in less than {duration}") def verify_single_file_content_in_directory(context: MinifiTestContext, content: str, directory: str, duration: str): - context.execute_steps(f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') - - -@then('in the "{container_name}" container at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') -def verify_at_least_one_file_content_in_container_directory(context: MinifiTestContext, container_name: str, content: str, directory: str, duration: str): + context.execute_steps( + f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container a single file with the content "{content}" is placed in the "{directory}" directory in less than {duration}' + ) + + +@then( + 'in the "{container_name}" container at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}' +) +def verify_at_least_one_file_content_in_container_directory( + context: MinifiTestContext, + container_name: str, + content: str, + directory: str, + duration: str, +): timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( condition=lambda: context.containers[container_name].directory_contains_file_with_content(directory, content), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[container_name].exited, context=context) + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[container_name].exited, + context=context, + ) @then('at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') @then("at least one file with the content '{content}' is placed in the '{directory}' directory in less than {duration}") -def verify_at_least_one_file_content_in_directory(context: MinifiTestContext, content: str, directory: str, duration: str): - context.execute_steps(f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}') +def verify_at_least_one_file_content_in_directory( + context: MinifiTestContext, content: str, directory: str, duration: str +): + context.execute_steps( + f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container at least one file with the content "{content}" is placed in the "{directory}" directory in less than {duration}' + ) @then("the logs of the '{container}' container do not contain the following message: '{message}' after {duration}") @then('the logs of the "{container}" container do not contain the following message: "{message}" after {duration}') -def verify_container_logs_do_not_contain_message(context: MinifiTestContext, container: str, message: str, duration: str): +def verify_container_logs_do_not_contain_message( + context: MinifiTestContext, container: str, message: str, duration: str +): duration_seconds = humanfriendly.parse_timespan(duration) time.sleep(duration_seconds) assert message not in context.containers[container].get_logs() or log_due_to_failure(context) @@ -73,7 +114,9 @@ def verify_container_logs_do_not_contain_message(context: MinifiTestContext, con @then('the Minifi logs do not contain the following message: "{message}" after {duration}') def verify_minifi_logs_do_not_contain_message(context: MinifiTestContext, message: str, duration: str): - context.execute_steps(f'then the logs of the "{DEFAULT_MINIFI_CONTAINER_NAME}" container do not contain the following message: "{message}" after {duration}') + context.execute_steps( + f'then the logs of the "{DEFAULT_MINIFI_CONTAINER_NAME}" container do not contain the following message: "{message}" after {duration}' + ) @then("the Minifi logs do not contain errors") @@ -83,55 +126,74 @@ def verify_minifi_logs_do_not_contain_errors(context: MinifiTestContext): @then("the Minifi logs do not contain warnings") def verify_minifi_logs_do_not_contain_warnings(context: MinifiTestContext): - assert "[warning]" not in context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs() or log_due_to_failure(context) + assert "[warning]" not in context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs() or log_due_to_failure( + context + ) @then("the logs of the '{container}' container contain the following message: '{message}' in less than {duration}") @then('the logs of the "{container}" container contain the following message: "{message}" in less than {duration}') def verify_container_logs_contain_message(context: MinifiTestContext, container: str, message: str, duration: str): duration_seconds = humanfriendly.parse_timespan(duration) - assert wait_for_condition(condition=lambda: message in context.containers[container].get_logs(), - timeout_seconds=duration_seconds, bail_condition=lambda: context.containers[container].exited, - context=context) + assert wait_for_condition( + condition=lambda: message in context.containers[container].get_logs(), + timeout_seconds=duration_seconds, + bail_condition=lambda: context.containers[container].exited, + context=context, + ) @then("the Minifi logs contain the following message: '{message}' in less than {duration}") @then('the Minifi logs contain the following message: "{message}" in less than {duration}') def verify_minifi_logs_contain_message(context: MinifiTestContext, message: str, duration: str): - context.execute_steps(f'then the logs of the "{DEFAULT_MINIFI_CONTAINER_NAME}" container contain the following message: "{message}" in less than {duration}') + context.execute_steps( + f'then the logs of the "{DEFAULT_MINIFI_CONTAINER_NAME}" container contain the following message: "{message}" in less than {duration}' + ) -@then("the Minifi logs contain the following message: \"{log_message}\" {count:d} times after {duration}") +@then('the Minifi logs contain the following message: "{log_message}" {count:d} times after {duration}') def verify_minifi_logs_contain_message_multiple_times(context, log_message, count, duration): duration_seconds = humanfriendly.parse_timespan(duration) time.sleep(duration_seconds) - assert context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs().count(log_message) == count or log_due_to_failure(context) + assert context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs().count( + log_message + ) == count or log_due_to_failure(context) -@then("the Minifi logs match the following regex: \"{regex}\" in less than {duration}") +@then('the Minifi logs match the following regex: "{regex}" in less than {duration}') def verify_minifi_logs_match_regex(context, regex, duration): duration_seconds = humanfriendly.parse_timespan(duration) - assert wait_for_condition(condition=lambda: re.search(regex, context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs()), - timeout_seconds=duration_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, - context=context) + assert wait_for_condition( + condition=lambda: re.search(regex, context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_logs()), + timeout_seconds=duration_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) @step('no errors were generated on the http-proxy regarding "{url}"') def verify_no_errors_on_http_proxy(context: MinifiTestContext, url: str): - http_proxy_container = next(container for container in context.containers.values() if isinstance(container, HttpProxy)) + http_proxy_container = next( + container for container in context.containers.values() if isinstance(container, HttpProxy) + ) assert http_proxy_container.check_http_proxy_access(url) or log_due_to_failure(context) @then('in the "{container}" container no files are placed in the "{directory}" directory in {duration} of running time') def verify_no_files_in_container_directory(context, container, directory, duration): duration_seconds = humanfriendly.parse_timespan(duration) - assert check_condition_after_wait(condition=lambda: context.containers[container].get_number_of_files(directory) == 0, - context=context, wait_time=duration_seconds) + assert check_condition_after_wait( + condition=lambda: context.containers[container].get_number_of_files(directory) == 0, + context=context, + wait_time=duration_seconds, + ) @then('no files are placed in the "{directory}" directory in {duration} of running time') def verify_no_files_in_directory(context, directory, duration): - context.execute_steps(f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container no files are placed in the "{directory}" directory in {duration} of running time') + context.execute_steps( + f'then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container no files are placed in the "{directory}" directory in {duration} of running time' + ) @then('{num:d} files are placed in the "{directory}" directory in less than {duration}') @@ -141,39 +203,69 @@ def verify_number_of_files_in_directory(context: MinifiTestContext, num: int, di if num == 0: context.execute_steps(f'then no files are placed in the "{directory}" directory in {duration} of running time') return - assert wait_for_condition(condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files(directory) == num, - timeout_seconds=duration_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, - context=context) + assert wait_for_condition( + condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files(directory) == num, + timeout_seconds=duration_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) @then('at least {num:d} files are placed in the "{directory}" directory in less than {duration}') @then('at least {num:d} file is placed in the "{directory}" directory in less than {duration}') def verify_at_least_number_of_files_in_directory(context: MinifiTestContext, num: int, directory: str, duration: str): duration_seconds = humanfriendly.parse_timespan(duration) - assert wait_for_condition(condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files(directory) >= num, - timeout_seconds=duration_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, - context=context) + assert wait_for_condition( + condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files(directory) >= num, + timeout_seconds=duration_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) @then('at least one file in "{directory}" content match the following regex: "{regex_str}" in less than {duration}') -@then('the content of at least one file in the "{directory}" directory matches the \'{regex_str}\' regex in less than {duration}') -def verify_file_content_matches_regex_in_directory(context: MinifiTestContext, directory: str, regex_str: str, duration: str): +@then( + "the content of at least one file in the \"{directory}\" directory matches the '{regex_str}' regex in less than {duration}" +) +def verify_file_content_matches_regex_in_directory( + context: MinifiTestContext, directory: str, regex_str: str, duration: str +): duration_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].directory_contains_file_with_regex(directory, regex_str), - timeout_seconds=duration_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, context=context) - - -@then('files with contents "{content_one}" and "{content_two}" are placed in the "{directory}" directory in less than {timeout}') -@then("files with contents '{content_one}' and '{content_two}' are placed in the '{directory}' directory in less than {timeout}") -def verify_files_with_two_contents_in_directory(context: MinifiTestContext, directory: str, timeout: str, content_one: str, content_two: str): + condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].directory_contains_file_with_regex( + directory, regex_str + ), + timeout_seconds=duration_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) + + +@then( + 'files with contents "{content_one}" and "{content_two}" are placed in the "{directory}" directory in less than {timeout}' +) +@then( + "files with contents '{content_one}' and '{content_two}' are placed in the '{directory}' directory in less than {timeout}" +) +def verify_files_with_two_contents_in_directory( + context: MinifiTestContext, + directory: str, + timeout: str, + content_one: str, + content_two: str, +): timeout_seconds = humanfriendly.parse_timespan(timeout) c1 = content_one.replace("\\n", "\n") c2 = content_two.replace("\\n", "\n") contents_arr = [c1, c2] - assert wait_for_condition(condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].verify_file_contents(directory, contents_arr), - timeout_seconds=timeout_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, - context=context) + assert wait_for_condition( + condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].verify_file_contents( + directory, contents_arr + ), + timeout_seconds=timeout_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) @then('files with contents "{contents}" are placed in the "{directory}" directory in less than {timeout}') @@ -181,31 +273,51 @@ def verify_files_with_contents_in_directory(context: MinifiTestContext, director timeout_seconds = humanfriendly.parse_timespan(timeout) new_contents = contents.replace("\\n", "\n") contents_arr = new_contents.split(",") - assert wait_for_condition(condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].verify_file_contents(directory, contents_arr), - timeout_seconds=timeout_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, - context=context) - - -@then('files with at least these contents "{contents}" are placed in the "{directory}" directory in less than {timeout}') -def verify_files_with_at_least_contents_in_directory(context: MinifiTestContext, directory: str, timeout: str, contents: str): + assert wait_for_condition( + condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].verify_file_contents( + directory, contents_arr + ), + timeout_seconds=timeout_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) + + +@then( + 'files with at least these contents "{contents}" are placed in the "{directory}" directory in less than {timeout}' +) +def verify_files_with_at_least_contents_in_directory( + context: MinifiTestContext, directory: str, timeout: str, contents: str +): timeout_seconds = humanfriendly.parse_timespan(timeout) new_contents = contents.replace("\\n", "\n") contents_arr = new_contents.split(",") - assert wait_for_condition(condition=lambda: all([context.containers[DEFAULT_MINIFI_CONTAINER_NAME].directory_contains_file_with_content(directory, content) for content in contents_arr]), - timeout_seconds=timeout_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, - context=context) + assert wait_for_condition( + condition=lambda: all( + context.containers[DEFAULT_MINIFI_CONTAINER_NAME].directory_contains_file_with_content(directory, content) + for content in contents_arr + ), + timeout_seconds=timeout_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) -@then("a file with the JSON content \"{content}\" is placed in the \"{directory}\" directory in less than {duration}") +@then('a file with the JSON content "{content}" is placed in the "{directory}" directory in less than {duration}') @then("a file with the JSON content '{content}' is placed in the '{directory}' directory in less than {duration}") def verify_file_with_json_content_in_directory(context: MinifiTestContext, content: str, directory: str, duration: str): timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].verify_path_with_json_content(directory, content), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, context=context) + condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].verify_path_with_json_content( + directory, content + ), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) -@then('MiNiFi\'s memory usage does not increase by more than {max_increase} after {duration}') +@then("MiNiFi's memory usage does not increase by more than {max_increase} after {duration}") def verify_minifi_memory_usage_increase(context: MinifiTestContext, max_increase: str, duration: str): time_in_seconds = humanfriendly.parse_timespan(duration) max_increase_in_bytes = humanfriendly.parse_size(max_increase) @@ -215,21 +327,45 @@ def verify_minifi_memory_usage_increase(context: MinifiTestContext, max_increase assert final_memory_usage - initial_memory_usage <= max_increase_in_bytes -@then("at least one file with the JSON content \"{content}\" is placed in the \"{directory}\" directory in less than {duration}") -@then("at least one file with the JSON content '{content}' is placed in the '{directory}' directory in less than {duration}") -def verify_at_least_one_file_with_json_content_in_directory(context: MinifiTestContext, content: str, directory: str, duration: str): +@then( + 'at least one file with the JSON content "{content}" is placed in the "{directory}" directory in less than {duration}' +) +@then( + "at least one file with the JSON content '{content}' is placed in the '{directory}' directory in less than {duration}" +) +def verify_at_least_one_file_with_json_content_in_directory( + context: MinifiTestContext, content: str, directory: str, duration: str +): timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( - condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].directory_contains_file_with_json_content(directory, content), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, context=context) - - -@then('after a wait of {duration}, at least {lower_bound:d} and at most {upper_bound:d} files are produced and placed in the "{directory}" directory') -def verify_file_count_bounds_in_directory(context: MinifiTestContext, lower_bound: int, upper_bound: int, duration: str, directory: str): + condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].directory_contains_file_with_json_content( + directory, content + ), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) + + +@then( + 'after a wait of {duration}, at least {lower_bound:d} and at most {upper_bound:d} files are produced and placed in the "{directory}" directory' +) +def verify_file_count_bounds_in_directory( + context: MinifiTestContext, + lower_bound: int, + upper_bound: int, + duration: str, + directory: str, +): duration_seconds = humanfriendly.parse_timespan(duration) - assert check_condition_after_wait(condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files(directory) >= lower_bound - and context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files(directory) <= upper_bound, - context=context, wait_time=duration_seconds) + assert check_condition_after_wait( + condition=lambda: ( + context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files(directory) >= lower_bound + and context.containers[DEFAULT_MINIFI_CONTAINER_NAME].get_number_of_files(directory) <= upper_bound + ), + context=context, + wait_time=duration_seconds, + ) @then('exactly these files are in the "{directory}" directory in less than {duration}: "{contents}"') @@ -239,42 +375,73 @@ def verify_exact_files_in_directory(context: MinifiTestContext, directory: str, return contents_arr = contents.split(",") timeout_in_seconds = humanfriendly.parse_timespan(duration) - assert wait_for_condition(condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].verify_file_contents(directory, contents_arr), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: False, - context=context) + assert wait_for_condition( + condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].verify_file_contents( + directory, contents_arr + ), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: False, + context=context, + ) @then('exactly these files are in the "{directory}" directory in less than {duration}: ""') -def verify_no_files_in_directory(context, directory, duration): +def verify_no_files_in_directory_exact(context, directory, duration): context.execute_steps(f'then no files are placed in the "{directory}" directory in {duration} of running time') -@then("at least one empty file is placed in the \"{directory}\" directory in less than {duration}") +@then('at least one empty file is placed in the "{directory}" directory in less than {duration}') def verify_at_least_one_empty_file_in_directory(context: MinifiTestContext, directory: str, duration: str): timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].directory_contains_empty_file(directory), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, context=context) - - -@then("in the \"{container_name}\" container at least one empty file is placed in the \"{directory}\" directory in less than {duration}") -def verify_at_least_one_empty_file_in_container_directory(context: MinifiTestContext, container_name: str, directory: str, duration: str): + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[DEFAULT_MINIFI_CONTAINER_NAME].exited, + context=context, + ) + + +@then( + 'in the "{container_name}" container at least one empty file is placed in the "{directory}" directory in less than {duration}' +) +def verify_at_least_one_empty_file_in_container_directory( + context: MinifiTestContext, container_name: str, directory: str, duration: str +): timeout_in_seconds = humanfriendly.parse_timespan(duration) assert wait_for_condition( condition=lambda: context.containers[container_name].directory_contains_empty_file(directory), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[container_name].exited, context=context) - - -@then("in the \"{container_name}\" container at least one file with minimum size of \"{size}\" is placed in the \"{directory}\" directory in less than {duration}") -def verify_file_with_minimum_size_in_container_directory(context: MinifiTestContext, container_name: str, directory: str, size: str, duration: str): + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[container_name].exited, + context=context, + ) + + +@then( + 'in the "{container_name}" container at least one file with minimum size of "{size}" is placed in the "{directory}" directory in less than {duration}' +) +def verify_file_with_minimum_size_in_container_directory( + context: MinifiTestContext, + container_name: str, + directory: str, + size: str, + duration: str, +): timeout_in_seconds = humanfriendly.parse_timespan(duration) size_in_bytes = humanfriendly.parse_size(size) assert wait_for_condition( - condition=lambda: context.containers[container_name].directory_contains_file_with_minimum_size(directory, size_in_bytes), - timeout_seconds=timeout_in_seconds, bail_condition=lambda: context.containers[container_name].exited, context=context) - - -@then("at least one file with minimum size of \"{size}\" is placed in the \"{directory}\" directory in less than {duration}") + condition=lambda: context.containers[container_name].directory_contains_file_with_minimum_size( + directory, size_in_bytes + ), + timeout_seconds=timeout_in_seconds, + bail_condition=lambda: context.containers[container_name].exited, + context=context, + ) + + +@then( + 'at least one file with minimum size of "{size}" is placed in the "{directory}" directory in less than {duration}' +) def verify_file_with_minimum_size_in_directory(context: MinifiTestContext, directory: str, size: str, duration: str): context.execute_steps( - f'Then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container at least one file with minimum size of "{size}" is placed in the "{directory}" directory in less than {duration}') + f'Then in the "{DEFAULT_MINIFI_CONTAINER_NAME}" container at least one file with minimum size of "{size}" is placed in the "{directory}" directory in less than {duration}' + ) diff --git a/behave_framework/src/minifi_behave/steps/configuration_steps.py b/behave_framework/src/minifi_behave/steps/configuration_steps.py index d2e8659c3c..fae80c0e1e 100644 --- a/behave_framework/src/minifi_behave/steps/configuration_steps.py +++ b/behave_framework/src/minifi_behave/steps/configuration_steps.py @@ -15,13 +15,14 @@ # limitations under the License. # -from behave import step, given - +from behave import given, step +from minifi_behave.containers.minifi_protocol import ( + conf_c2_flow_url, + enable_log_metrics_publisher, + enable_openssl_fips_mode, + set_up_ssl_properties, +) from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.containers.minifi_protocol import enable_openssl_fips_mode -from minifi_behave.containers.minifi_protocol import set_up_ssl_properties -from minifi_behave.containers.minifi_protocol import enable_log_metrics_publisher -from minifi_behave.containers.minifi_protocol import conf_c2_flow_url @step('MiNiFi configuration "{config_key}" is set to "{config_value}"') diff --git a/behave_framework/src/minifi_behave/steps/core_steps.py b/behave_framework/src/minifi_behave/steps/core_steps.py index 2c569637bb..15e99115f2 100644 --- a/behave_framework/src/minifi_behave/steps/core_steps.py +++ b/behave_framework/src/minifi_behave/steps/core_steps.py @@ -16,20 +16,24 @@ # import logging +import os import random import string -import os import time import uuid import humanfriendly -from behave import when, step, given - -from minifi_behave.containers.http_proxy_container import HttpProxy -from minifi_behave.containers.nifi_container import NifiContainer +from behave import given, step, when from minifi_behave.containers.directory import Directory from minifi_behave.containers.file import File -from minifi_behave.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext +from minifi_behave.containers.http_proxy_container import HttpProxy +from minifi_behave.containers.nifi_container import NifiContainer +from minifi_behave.core.minifi_test_context import ( + DEFAULT_MINIFI_CONTAINER_NAME, + MinifiTestContext, +) + +logger = logging.getLogger(__name__) @when("both instances start up") @@ -37,19 +41,19 @@ def start_all_instances(context: MinifiTestContext): for container in context.containers.values(): assert container.deploy(context) - logging.debug("All instances started up") + logger.debug("All instances started up") @when("the MiNiFi instance starts up") def start_minifi_instance(context: MinifiTestContext): assert context.get_or_create_default_minifi_container().deploy(context) - logging.debug("MiNiFi instance started up") + logger.debug("MiNiFi instance started up") @step('a directory at "{directory}" has a file with the size "{size}"') def create_file_with_size_in_directory(context: MinifiTestContext, directory: str, size: str): size = humanfriendly.parse_size(size) - content = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(size)) + content = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(size)) dirs = context.get_or_create_default_minifi_container().dirs if directory in dirs: dirs[directory].files[str(uuid.uuid4())] = content @@ -59,7 +63,13 @@ def create_file_with_size_in_directory(context: MinifiTestContext, directory: st dirs.append(new_dir) -def __add_directory_with_file_to_container(context: MinifiTestContext, directory: str, file_name: str, content: str | bytes, container_name: str): +def __add_directory_with_file_to_container( + context: MinifiTestContext, + directory: str, + file_name: str, + content: str | bytes, + container_name: str, +): dirs = context.get_or_create_minifi_container(container_name).dirs if isinstance(content, str): content = content.replace("\\n", "\n") @@ -73,14 +83,18 @@ def __add_directory_with_file_to_container(context: MinifiTestContext, directory @step('a directory at "{directory}" has a file with the content "{content}" in the "{flow_name}" flow') @step("a directory at '{directory}' has a file with the content '{content}' in the '{flow_name}' flow") -def create_file_with_content_in_directory_for_flow(context: MinifiTestContext, directory: str, content: str, flow_name: str): +def create_file_with_content_in_directory_for_flow( + context: MinifiTestContext, directory: str, content: str, flow_name: str +): __add_directory_with_file_to_container(context, directory, str(uuid.uuid4()), content, flow_name) @step('a directory at "{directory}" has a file with the content "{content}"') @step("a directory at '{directory}' has a file with the content '{content}'") def create_file_with_content_in_directory(context: MinifiTestContext, directory: str, content: str): - context.execute_steps(f'given a directory at "{directory}" has a file with the content "{content}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') + context.execute_steps( + f'given a directory at "{directory}" has a file with the content "{content}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) @step('a directory at "{directory}" has a file with the content from "{path}"') @@ -90,11 +104,15 @@ def create_file_with_content_from_path_in_directory(context: MinifiTestContext, content = None with open(context.resource_dir / path, "rb") as f: content = f.read() - __add_directory_with_file_to_container(context, directory, str(uuid.uuid4()), content, DEFAULT_MINIFI_CONTAINER_NAME) + __add_directory_with_file_to_container( + context, directory, str(uuid.uuid4()), content, DEFAULT_MINIFI_CONTAINER_NAME + ) @step('a directory at "{directory}" has a file "{file_name}" with the content "{content}"') -def create_file_with_name_and_content_in_directory(context: MinifiTestContext, directory: str, file_name: str, content: str): +def create_file_with_name_and_content_in_directory( + context: MinifiTestContext, directory: str, file_name: str, content: str +): __add_directory_with_file_to_container(context, directory, file_name, content, DEFAULT_MINIFI_CONTAINER_NAME) @@ -105,15 +123,21 @@ def file_with_name_and_content_present_in_path(context: MinifiTestContext, file_ @given('a file with the content "{content}" is present in "{path}" in the "{container_name}" flow') -def file_with_content_present_in_path_for_flow(context: MinifiTestContext, content: str, path: str, container_name: str): +def file_with_content_present_in_path_for_flow( + context: MinifiTestContext, content: str, path: str, container_name: str +): new_content = content.replace("\\n", "\n") - context.get_or_create_minifi_container(container_name).files.append(File(os.path.join(path, str(uuid.uuid4())), new_content)) + context.get_or_create_minifi_container(container_name).files.append( + File(os.path.join(path, str(uuid.uuid4())), new_content) + ) @given('a file with the content "{content}" is present in "{path}"') @given("a file with the content '{content}' is present in '{path}'") def file_with_content_present_in_path(context: MinifiTestContext, content: str, path: str): - context.execute_steps(f"given a file with the content \"{content}\" is present in \"{path}\" in the \"{DEFAULT_MINIFI_CONTAINER_NAME}\" flow") + context.execute_steps( + f'given a file with the content "{content}" is present in "{path}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) @when('a file with the content "{content}" is placed in "{path}" in the "{container_name}" flow') @@ -124,23 +148,31 @@ def place_file_with_content_in_path_for_flow(context: MinifiTestContext, content @when('a file with the content "{content}" is placed in "{path}"') def place_file_with_content_in_path(context: MinifiTestContext, content: str, path: str): - context.execute_steps(f"when a file with the content \"{content}\" is placed in \"{path}\" in the \"{DEFAULT_MINIFI_CONTAINER_NAME}\" flow") + context.execute_steps( + f'when a file with the content "{content}" is placed in "{path}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) -@given("an empty file is present in \"{path}\"") +@given('an empty file is present in "{path}"') def empty_file_present_in_path(context, path): context.get_or_create_default_minifi_container().files.append(File(os.path.join(path, str(uuid.uuid4())), "")) -@given('a host resource file "{filename}" is bound to the "{container_path}" path in the MiNiFi container "{container_name}"') -def bind_host_resource_file_to_container_path_for_container(context: MinifiTestContext, filename: str, container_path: str, container_name: str): +@given( + 'a host resource file "{filename}" is bound to the "{container_path}" path in the MiNiFi container "{container_name}"' +) +def bind_host_resource_file_to_container_path_for_container( + context: MinifiTestContext, filename: str, container_path: str, container_name: str +): path = os.path.join(context.resource_dir, filename) context.get_or_create_minifi_container(container_name).add_host_file(path, container_path) @given('a host resource file "{filename}" is bound to the "{container_path}" path in the MiNiFi container') def bind_host_resource_file_to_container_path(context: MinifiTestContext, filename: str, container_path: str): - context.execute_steps(f"given a host resource file \"{filename}\" is bound to the \"{container_path}\" path in the MiNiFi container \"{DEFAULT_MINIFI_CONTAINER_NAME}\"") + context.execute_steps( + f'given a host resource file "{filename}" is bound to the "{container_path}" path in the MiNiFi container "{DEFAULT_MINIFI_CONTAINER_NAME}"' + ) @step("after {duration} have passed") @@ -155,7 +187,7 @@ def stop_minifi(context: MinifiTestContext): context.get_or_create_default_minifi_container().stop() -@when("the \"{container_name}\" flow is stopped") +@when('the "{container_name}" flow is stopped') def stop_flow(context: MinifiTestContext, container_name: str): context.get_or_create_minifi_container(container_name).stop() @@ -165,17 +197,17 @@ def restart_minifi(context: MinifiTestContext): context.get_or_create_default_minifi_container().restart() -@when("the \"{container_name}\" flow is restarted") +@when('the "{container_name}" flow is restarted') def restart_flow(context: MinifiTestContext, container_name: str): context.get_or_create_minifi_container(container_name).restart() -@when("the \"{container_name}\" flow is started") +@when('the "{container_name}" flow is started') def start_flow(context: MinifiTestContext, container_name: str): context.get_or_create_minifi_container(container_name).start() -@when("the \"{container_name}\" flow is killed") +@when('the "{container_name}" flow is killed') def kill_flow(context: MinifiTestContext, container_name: str): context.get_or_create_minifi_container(container_name).kill() @@ -195,7 +227,7 @@ def setup_nifi_container_with_ssl(context: MinifiTestContext): context.containers["nifi"] = NifiContainer(context, use_ssl=True) -@when('NiFi is started') +@when("NiFi is started") def start_nifi(context): assert context.containers["nifi"].deploy(context) or context.containers["nifi"].log_app_output() diff --git a/behave_framework/src/minifi_behave/steps/flow_building_steps.py b/behave_framework/src/minifi_behave/steps/flow_building_steps.py index b880bbfbc0..a7f737a558 100644 --- a/behave_framework/src/minifi_behave/steps/flow_building_steps.py +++ b/behave_framework/src/minifi_behave/steps/flow_building_steps.py @@ -17,9 +17,12 @@ import logging import uuid -from behave import given, step -from minifi_behave.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext +from behave import given, step +from minifi_behave.core.minifi_test_context import ( + DEFAULT_MINIFI_CONTAINER_NAME, + MinifiTestContext, +) from minifi_behave.minifi.connection import Connection from minifi_behave.minifi.controller_service import ControllerService from minifi_behave.minifi.funnel import Funnel @@ -27,6 +30,8 @@ from minifi_behave.minifi.parameter_context import ParameterContext from minifi_behave.minifi.processor import Processor +logger = logging.getLogger(__name__) + @given("a MiNiFi CPP server with yaml config") def minifi_cpp_server_with_yaml_config(context: MinifiTestContext): @@ -35,42 +40,77 @@ def minifi_cpp_server_with_yaml_config(context: MinifiTestContext): @given("a transient MiNiFi flow with a LogOnDestructionProcessor processor") def transient_flow_with_logondestructionprocessor(context: MinifiTestContext): - context.get_or_create_default_minifi_container().command = ["/bin/sh", "-c", "timeout 10s ./bin/minifi.sh run && sleep 100"] + context.get_or_create_default_minifi_container().command = [ + "/bin/sh", + "-c", + "timeout 10s ./bin/minifi.sh run && sleep 100", + ] context.get_or_create_default_minifi_container().flow_definition.add_processor( - Processor("LogOnDestructionProcessor", "LogOnDestructionProcessor")) + Processor("LogOnDestructionProcessor", "LogOnDestructionProcessor") + ) @given( - 'a {processor_type} processor with the name "{processor_name}" and the "{property_name}" property set to "{property_value}"') -def processor_with_name_and_property(context: MinifiTestContext, processor_type: str, processor_name: str, property_name: str, - property_value: str): + 'a {processor_type} processor with the name "{processor_name}" and the "{property_name}" property set to "{property_value}"' +) +def processor_with_name_and_property( + context: MinifiTestContext, + processor_type: str, + processor_name: str, + property_name: str, + property_value: str, +): processor = Processor(processor_type, processor_name) processor.add_property(property_name, property_value) context.get_or_create_default_minifi_container().flow_definition.add_processor(processor) @step('a {processor_type} processor with the "{property_name}" property set to "{property_value}"') -def processor_with_property(context: MinifiTestContext, processor_type: str, property_name: str, property_value: str): +def processor_with_property( + context: MinifiTestContext, + processor_type: str, + property_name: str, + property_value: str, +): context.execute_steps( - f'Given a {processor_type} processor with the name "{processor_type}" and the "{property_name}" property set to "{property_value}"') + f'Given a {processor_type} processor with the name "{processor_type}" and the "{property_name}" property set to "{property_value}"' + ) -@step('a {processor_type} processor with the "{property_name}" property set to "{property_value}" in the "{minifi_container_name}" flow') -def processor_with_property_in_minifi_flow(context: MinifiTestContext, processor_type: str, property_name: str, property_value: str, minifi_container_name: str): +@step( + 'a {processor_type} processor with the "{property_name}" property set to "{property_value}" in the "{minifi_container_name}" flow' +) +def processor_with_property_in_minifi_flow( + context: MinifiTestContext, + processor_type: str, + property_name: str, + property_value: str, + minifi_container_name: str, +): processor = Processor(processor_type, processor_type) processor.add_property(property_name, property_value) context.get_or_create_minifi_container(minifi_container_name).flow_definition.add_processor(processor) @step('a {processor_type} processor with the "{property_name}" property set to "{property_value}" in the NiFi flow') -def processor_with_property_in_nifi_flow(context: MinifiTestContext, processor_type: str, property_name: str, property_value: str): +def processor_with_property_in_nifi_flow( + context: MinifiTestContext, + processor_type: str, + property_name: str, + property_value: str, +): processor = Processor(processor_type, processor_type) processor.add_property(property_name, property_value) context.containers["nifi"].flow_definition.add_processor(processor) @given('a {processor_type} processor with the name "{processor_name}" in the "{minifi_container_name}" flow') -def processor_with_name_in_minifi_flow(context: MinifiTestContext, processor_type: str, processor_name: str, minifi_container_name: str): +def processor_with_name_in_minifi_flow( + context: MinifiTestContext, + processor_type: str, + processor_name: str, + minifi_container_name: str, +): processor = Processor(processor_type, processor_name) context.get_or_create_minifi_container(minifi_container_name).flow_definition.add_processor(processor) @@ -78,10 +118,11 @@ def processor_with_name_in_minifi_flow(context: MinifiTestContext, processor_typ @given('a {processor_type} processor with the name "{processor_name}"') def processor_with_name(context: MinifiTestContext, processor_type: str, processor_name: str): context.execute_steps( - f'given a {processor_type} processor with the name "{processor_name}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') + f'given a {processor_type} processor with the name "{processor_name}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) -@given("a {processor_type} processor in the \"{minifi_container_name}\" flow") +@given('a {processor_type} processor in the "{minifi_container_name}" flow') def processor_in_minifi_flow(context: MinifiTestContext, processor_type: str, minifi_container_name: str): processor = Processor(processor_type, processor_type) context.get_or_create_minifi_container(minifi_container_name).flow_definition.add_processor(processor) @@ -98,9 +139,19 @@ def processor_setup(context: MinifiTestContext, processor_type: str): context.execute_steps(f'given a {processor_type} processor in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') -@given('the "{property_name}" property of the "{port_name}" port in the "{rpg_name}" remote process group is set to "{property_value}"') -def set_property_of_port_in_rpg(context: MinifiTestContext, property_name: str, port_name: str, rpg_name: str, property_value: str): - rpg = context.get_or_create_minifi_container(DEFAULT_MINIFI_CONTAINER_NAME).flow_definition.get_remote_process_group(rpg_name) +@given( + 'the "{property_name}" property of the "{port_name}" port in the "{rpg_name}" remote process group is set to "{property_value}"' +) +def set_property_of_port_in_rpg( + context: MinifiTestContext, + property_name: str, + port_name: str, + rpg_name: str, + property_value: str, +): + rpg = context.get_or_create_minifi_container( + DEFAULT_MINIFI_CONTAINER_NAME + ).flow_definition.get_remote_process_group(rpg_name) if not rpg: raise ValueError(f"Remote Process Group with name {rpg_name} not found") port = rpg.get_input_port(port_name) @@ -109,9 +160,19 @@ def set_property_of_port_in_rpg(context: MinifiTestContext, property_name: str, port.add_property(property_name, property_value) -@given('the "{property_name}" property of the {processor_name} processor is set to "{property_value}" in the "{minifi_container_name}" flow') -def set_property_of_processor_in_minifi_flow(context: MinifiTestContext, property_name: str, processor_name: str, property_value: str, minifi_container_name: str): - processor = context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor(processor_name) +@given( + 'the "{property_name}" property of the {processor_name} processor is set to "{property_value}" in the "{minifi_container_name}" flow' +) +def set_property_of_processor_in_minifi_flow( + context: MinifiTestContext, + property_name: str, + processor_name: str, + property_value: str, + minifi_container_name: str, +): + processor = context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor( + processor_name + ) if property_value == "(not set)": processor.remove_property(property_name) else: @@ -119,7 +180,12 @@ def set_property_of_processor_in_minifi_flow(context: MinifiTestContext, propert @given('the "{property_name}" property of the {processor_name} processor is set to "{property_value}" in the NiFi flow') -def set_property_of_processor_in_nifi_flow(context: MinifiTestContext, property_name: str, processor_name: str, property_value: str): +def set_property_of_processor_in_nifi_flow( + context: MinifiTestContext, + property_name: str, + processor_name: str, + property_value: str, +): processor = context.containers["nifi"].flow_definition.get_processor(processor_name) if property_value == "(not set)": processor.remove_property(property_name) @@ -128,14 +194,27 @@ def set_property_of_processor_in_nifi_flow(context: MinifiTestContext, property_ @given('the "{property_name}" property of the {processor_name} processor is set to "{property_value}"') -def set_property_of_processor(context: MinifiTestContext, property_name: str, processor_name: str, property_value: str): +def set_property_of_processor( + context: MinifiTestContext, + property_name: str, + processor_name: str, + property_value: str, +): context.execute_steps( - f'given the "{property_name}" property of the {processor_name} processor is set to "{property_value}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') + f'given the "{property_name}" property of the {processor_name} processor is set to "{property_value}" in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) @step('the "{property_name}" property of the {controller_name} controller service is set to "{property_value}"') -def set_property_of_controller_service(context: MinifiTestContext, property_name: str, controller_name: str, property_value: str): - controller_service = context.get_or_create_default_minifi_container().flow_definition.get_controller_service(controller_name) +def set_property_of_controller_service( + context: MinifiTestContext, + property_name: str, + controller_name: str, + property_value: str, +): + controller_service = context.get_or_create_default_minifi_container().flow_definition.get_controller_service( + controller_name + ) if property_value == "(not set)": controller_service.remove_property(property_name) else: @@ -147,9 +226,19 @@ def setup_funnel(context: MinifiTestContext, funnel_name: str): context.get_or_create_default_minifi_container().flow_definition.add_funnel(Funnel(funnel_name)) -@step('in the "{minifi_container_name}" flow the "{relationship_name}" relationship of the {source} processor is connected to the {target}') -@step('in the "{minifi_container_name}" flow the "{relationship_name}" relationship of the {source} node is connected to the {target}') -def connect_relationship_in_minifi_flow(context: MinifiTestContext, relationship_name: str, source: str, target: str, minifi_container_name: str): +@step( + 'in the "{minifi_container_name}" flow the "{relationship_name}" relationship of the {source} processor is connected to the {target}' +) +@step( + 'in the "{minifi_container_name}" flow the "{relationship_name}" relationship of the {source} node is connected to the {target}' +) +def connect_relationship_in_minifi_flow( + context: MinifiTestContext, + relationship_name: str, + source: str, + target: str, + minifi_container_name: str, +): connection = Connection(source_name=source, source_relationship=relationship_name, target_name=target) context.get_or_create_minifi_container(minifi_container_name).flow_definition.add_connection(connection) @@ -163,12 +252,18 @@ def connect_relationship_in_nifi_flow(context: MinifiTestContext, relationship_n @step('the "{relationship_name}" relationship of the {source} processor is connected to the {target}') @step('the "{relationship_name}" relationship of the {source} node is connected to the {target}') def connect_relationship(context: MinifiTestContext, relationship_name: str, source: str, target: str): - context.execute_steps(f'given in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow the "{relationship_name}" relationship of the {source} processor is connected to the {target}') + context.execute_steps( + f'given in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow the "{relationship_name}" relationship of the {source} processor is connected to the {target}' + ) -@step("the output port \"{port_name}\" is connected to the {destination_name} processor") +@step('the output port "{port_name}" is connected to the {destination_name} processor') def connect_output_port_to_processor(context: MinifiTestContext, port_name: str, destination_name: str): - connection = Connection(source_name=port_name, source_relationship="undefined", target_name=destination_name) + connection = Connection( + source_name=port_name, + source_relationship="undefined", + target_name=destination_name, + ) context.get_or_create_minifi_container(DEFAULT_MINIFI_CONTAINER_NAME).flow_definition.add_connection(connection) @@ -178,31 +273,54 @@ def connect_funnel_to_target(context: MinifiTestContext, funnel_name: str, targe context.get_or_create_default_minifi_container().flow_definition.add_connection(connection) -@step("{processor_name}'s {relationship} relationship is auto-terminated in the \"{minifi_container_name}\" flow") -def auto_terminate_relationship_in_minifi_flow(context: MinifiTestContext, processor_name: str, relationship: str, minifi_container_name: str): - context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor(processor_name).auto_terminated_relationships.append( - relationship) +@step('{processor_name}\'s {relationship} relationship is auto-terminated in the "{minifi_container_name}" flow') +def auto_terminate_relationship_in_minifi_flow( + context: MinifiTestContext, + processor_name: str, + relationship: str, + minifi_container_name: str, +): + context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor( + processor_name + ).auto_terminated_relationships.append(relationship) @step("{processor_name}'s {relationship} relationship is auto-terminated in the NiFi flow") def auto_terminate_relationship_in_nifi_flow(context: MinifiTestContext, processor_name: str, relationship: str): - context.containers["nifi"].flow_definition.get_processor(processor_name).auto_terminated_relationships.append(relationship) + context.containers["nifi"].flow_definition.get_processor(processor_name).auto_terminated_relationships.append( + relationship + ) @step("{processor_name}'s {relationship} relationship is auto-terminated") -@step("the \"{relationship}\" relationship of the {processor_name} processor is auto-terminated") +@step('the "{relationship}" relationship of the {processor_name} processor is auto-terminated') def auto_terminate_relationship(context: MinifiTestContext, processor_name: str, relationship: str): - context.execute_steps(f'given {processor_name}\'s {relationship} relationship is auto-terminated in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') + context.execute_steps( + f'given {processor_name}\'s {relationship} relationship is auto-terminated in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) @given("a transient MiNiFi flow is set up") def setup_transient_minifi_flow(context: MinifiTestContext): - context.get_or_create_default_minifi_container().command = ["/bin/sh", "-c", "timeout 10s ./bin/minifi.sh run && sleep 100"] + context.get_or_create_default_minifi_container().command = [ + "/bin/sh", + "-c", + "timeout 10s ./bin/minifi.sh run && sleep 100", + ] -@step('the scheduling period of the {processor_name} processor is set to "{duration_str}" in the "{minifi_container_name}" flow') -def set_scheduling_period_in_minifi_flow(context: MinifiTestContext, processor_name: str, duration_str: str, minifi_container_name: str): - context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor(processor_name).scheduling_period = duration_str +@step( + 'the scheduling period of the {processor_name} processor is set to "{duration_str}" in the "{minifi_container_name}" flow' +) +def set_scheduling_period_in_minifi_flow( + context: MinifiTestContext, + processor_name: str, + duration_str: str, + minifi_container_name: str, +): + context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor( + processor_name + ).scheduling_period = duration_str @step('the scheduling period of the {processor_name} processor is set to "{duration_str}" in the NiFi flow') @@ -212,38 +330,58 @@ def set_scheduling_period_in_nifi_flow(context: MinifiTestContext, processor_nam @step('the scheduling period of the {processor_name} processor is set to "{duration_str}"') def set_scheduling_period(context: MinifiTestContext, processor_name: str, duration_str: str): - context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name).scheduling_period = duration_str + context.get_or_create_default_minifi_container().flow_definition.get_processor( + processor_name + ).scheduling_period = duration_str @given("parameter context name is set to '{context_name}'") def add_parameter_context(context: MinifiTestContext, context_name: str): - context.get_or_create_default_minifi_container().flow_definition.parameter_contexts.append(ParameterContext(context_name)) + context.get_or_create_default_minifi_container().flow_definition.parameter_contexts.append( + ParameterContext(context_name) + ) @step( - "a non-sensitive parameter in the flow config called '{parameter_name}' with the value '{parameter_value}' in the parameter context '{context_name}'") -def add_non_sensitive_parameter(context: MinifiTestContext, parameter_name: str, parameter_value: str, context_name: str): - parameter_context = context.get_or_create_default_minifi_container().flow_definition.get_parameter_context(context_name) + "a non-sensitive parameter in the flow config called '{parameter_name}' with the value '{parameter_value}' in the parameter context '{context_name}'" +) +def add_non_sensitive_parameter( + context: MinifiTestContext, + parameter_name: str, + parameter_value: str, + context_name: str, +): + parameter_context = context.get_or_create_default_minifi_container().flow_definition.get_parameter_context( + context_name + ) parameter_context.parameters.append(Parameter(parameter_name, parameter_value, False)) -@given("these processor properties are set in the \"{minifi_container_name}\" flow") +@given('these processor properties are set in the "{minifi_container_name}" flow') def set_processor_properties_in_minifi_flow(context: MinifiTestContext, minifi_container_name: str): for row in context.table: - processor = context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor(row["processor name"]) + processor = context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor( + row["processor name"] + ) processor.add_property(row["property name"], row["property value"]) @given("these processor properties are set") def set_processor_properties(context: MinifiTestContext): for row in context.table: - processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(row["processor name"]) + processor = context.get_or_create_default_minifi_container().flow_definition.get_processor( + row["processor name"] + ) processor.add_property(row["property name"], row["property value"]) -@given("a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration in the \"{container_name}\" flow") +@given( + 'a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration in the "{container_name}" flow' +) def step_impl(context: MinifiTestContext, proxy_type: str, container_name: str): - controller_service = ControllerService(class_name="ProxyConfigurationService", service_name="ProxyConfigurationService") + controller_service = ControllerService( + class_name="ProxyConfigurationService", service_name="ProxyConfigurationService" + ) prefix = proxy_type.lower() if proxy_type.lower() != "direct" else "http" controller_service.add_property("Proxy Server Host", f"{prefix}://http-proxy-{context.scenario_id}") controller_service.add_property("Proxy User Name", "admin") @@ -253,12 +391,16 @@ def step_impl(context: MinifiTestContext, proxy_type: str, container_name: str): controller_service.add_property("Proxy Server Port", "3128") else: controller_service.add_property("Proxy Server Port", "3129") - context.get_or_create_minifi_container(container_name).flow_definition.controller_services.append(controller_service) + context.get_or_create_minifi_container(container_name).flow_definition.controller_services.append( + controller_service + ) @given("a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration") -def step_impl(context: MinifiTestContext, proxy_type: str): - context.execute_steps(f"given a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration in the \"{DEFAULT_MINIFI_CONTAINER_NAME}\" flow") +def set_up_proxy_configuration_service_in_default_flow(context: MinifiTestContext, proxy_type: str): + context.execute_steps( + f'given a ProxyConfigurationService controller service is set up with {proxy_type} proxy configuration in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) @step("the processors are connected up as described here") @@ -269,16 +411,24 @@ def connect_processors_from_table(context: MinifiTestContext): relationship = row["relationship name"] if dest_proc_name == "auto-terminated": context.get_or_create_default_minifi_container().flow_definition.get_processor( - source_proc_name).auto_terminated_relationships.append(relationship) + source_proc_name + ).auto_terminated_relationships.append(relationship) else: - connection = Connection(source_name=row["source name"], source_relationship=relationship, - target_name=row["destination name"]) + connection = Connection( + source_name=row["source name"], + source_relationship=relationship, + target_name=row["destination name"], + ) context.get_or_create_default_minifi_container().flow_definition.add_connection(connection) -@step("{processor_name} is EVENT_DRIVEN in the \"{minifi_container_name}\" flow") -def set_processor_event_driven_in_minifi_flow(context: MinifiTestContext, processor_name: str, minifi_container_name: str): - processor = context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor(processor_name) +@step('{processor_name} is EVENT_DRIVEN in the "{minifi_container_name}" flow') +def set_processor_event_driven_in_minifi_flow( + context: MinifiTestContext, processor_name: str, minifi_container_name: str +): + processor = context.get_or_create_minifi_container(minifi_container_name).flow_definition.get_processor( + processor_name + ) processor.scheduling_strategy = "EVENT_DRIVEN" @@ -301,30 +451,51 @@ def setup_controller_service(context: MinifiTestContext, service_name: str): context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) -@given('a {class_name} controller service named "{service_name}" is set up and the "{property_name}" property set to "{property_value}"') -def setup_controller_service_with_property(context: MinifiTestContext, class_name: str, service_name: str, property_name: str, property_value: str): +@given( + 'a {class_name} controller service named "{service_name}" is set up and the "{property_name}" property set to "{property_value}"' +) +def setup_named_controller_service_with_property( + context: MinifiTestContext, + class_name: str, + service_name: str, + property_name: str, + property_value: str, +): controller_service = ControllerService(class_name=class_name, service_name=service_name) controller_service.add_property(property_name, property_value) context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) @given('a {service_name} controller service is set up and the "{property_name}" property set to "{property_value}"') -def setup_controller_service_with_property(context: MinifiTestContext, service_name: str, property_name: str, property_value: str): +def setup_controller_service_with_property( + context: MinifiTestContext, + service_name: str, + property_name: str, + property_value: str, +): controller_service = ControllerService(class_name=service_name, service_name=service_name) controller_service.add_property(property_name, property_value) context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) -@given("the \"{property_name}\" property of the {processor_name} processor is set to match the attribute \"{attribute_key}\" to \"{attribute_value}\"") -def set_processor_property_to_match_attribute(context: MinifiTestContext, property_name: str, processor_name: str, attribute_key: str, attribute_value: str): +@given( + 'the "{property_name}" property of the {processor_name} processor is set to match the attribute "{attribute_key}" to "{attribute_value}"' +) +def set_processor_property_to_match_attribute( + context: MinifiTestContext, + property_name: str, + processor_name: str, + attribute_key: str, + attribute_value: str, +): processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name) if attribute_value == "(not set)": # Ignore filtering processor.add_property(property_name, "true") return filtering = "${" + attribute_key + ":equals('" + attribute_value + "')}" - logging.info("Filter: \"%s\"", filtering) - logging.info("Key: \"%s\", value: \"%s\"", attribute_key, attribute_value) + logger.info('Filter: "%s"', filtering) + logger.info('Key: "%s", value: "%s"', attribute_key, attribute_value) processor.add_property(property_name, filtering) @@ -334,16 +505,24 @@ def set_processor_max_concurrent_tasks(context, processor_name: str, max_concurr processor.set_max_concurrent_tasks(max_concurrent_tasks) -@given("the \"{property_name}\" properties of the {processor_name_one} and {processor_name_two} processors are set to the same random UUID") +@given( + 'the "{property_name}" properties of the {processor_name_one} and {processor_name_two} processors are set to the same random UUID' +) def set_processors_properties_to_same_uuid(context, property_name, processor_name_one, processor_name_two): uuid_str = str(uuid.uuid4()) - context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name_one).add_property(property_name, uuid_str) - context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name_two).add_property(property_name, uuid_str) + context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name_one).add_property( + property_name, uuid_str + ) + context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name_two).add_property( + property_name, uuid_str + ) # TLS def add_ssl_context_service_for_minifi(context: MinifiTestContext, cert_name: str, use_system_cert_store: bool = False): - ssl_context_service = context.get_or_create_default_minifi_container().flow_definition.get_controller_service("SSLContextService") + ssl_context_service = context.get_or_create_default_minifi_container().flow_definition.get_controller_service( + "SSLContextService" + ) if ssl_context_service is not None: return controller_service = ControllerService(class_name="SSLContextService", service_name="SSLContextService") @@ -366,58 +545,82 @@ def setup_ssl_context_service(context: MinifiTestContext): def setup_ssl_context_service_for_processor(context, processor_name): add_ssl_context_service_for_minifi(context, "minifi_client") processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name) - processor.add_property('SSL Context Service', 'SSLContextService') + processor.add_property("SSL Context Service", "SSLContextService") @given("an ssl context service using the system CA cert store is set up for {processor_name}") def setup_ssl_context_service_system_ca_for_processor(context: MinifiTestContext, processor_name): add_ssl_context_service_for_minifi(context, "minifi_client", use_system_cert_store=True) processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name) - processor.add_property('SSL Context Service', 'SSLContextService') + processor.add_property("SSL Context Service", "SSLContextService") -@given("a RemoteProcessGroup node with name \"{rpg_name}\" is opened on \"{address}\" with transport protocol set to \"{transport_protocol}\"") +@given( + 'a RemoteProcessGroup node with name "{rpg_name}" is opened on "{address}" with transport protocol set to "{transport_protocol}"' +) def open_rpg_with_transport_protocol(context: MinifiTestContext, rpg_name: str, address: str, transport_protocol: str): - context.get_or_create_default_minifi_container().flow_definition.add_remote_process_group(address, rpg_name, transport_protocol) + context.get_or_create_default_minifi_container().flow_definition.add_remote_process_group( + address, rpg_name, transport_protocol + ) -@given("a RemoteProcessGroup node with name \"{rpg_name}\" is opened on \"{address}\"") +@given('a RemoteProcessGroup node with name "{rpg_name}" is opened on "{address}"') def open_rpg(context: MinifiTestContext, rpg_name: str, address: str): - context.execute_steps(f"given a RemoteProcessGroup node with name \"{rpg_name}\" is opened on \"{address}\" with transport protocol set to \"RAW\"") + context.execute_steps( + f'given a RemoteProcessGroup node with name "{rpg_name}" is opened on "{address}" with transport protocol set to "RAW"' + ) -@given("an input port with name \"{port_name}\" is created on the RemoteProcessGroup named \"{rpg_name}\"") +@given('an input port with name "{port_name}" is created on the RemoteProcessGroup named "{rpg_name}"') def create_input_port_on_rpg(context: MinifiTestContext, port_name: str, rpg_name: str): context.get_or_create_default_minifi_container().flow_definition.add_input_port_to_rpg(rpg_name, port_name) -@given("an input port using compression with name \"{port_name}\" is created on the RemoteProcessGroup named \"{rpg_name}\"") +@given( + 'an input port using compression with name "{port_name}" is created on the RemoteProcessGroup named "{rpg_name}"' +) def create_input_port_with_compression_on_rpg(context: MinifiTestContext, port_name: str, rpg_name: str): - context.get_or_create_default_minifi_container().flow_definition.add_input_port_to_rpg(rpg_name, port_name, use_compression=True) + context.get_or_create_default_minifi_container().flow_definition.add_input_port_to_rpg( + rpg_name, port_name, use_compression=True + ) -@given("an output port with name \"{port_name}\" is created on the RemoteProcessGroup named \"{rpg_name}\"") +@given('an output port with name "{port_name}" is created on the RemoteProcessGroup named "{rpg_name}"') def create_output_port_on_rpg(context: MinifiTestContext, port_name: str, rpg_name: str): context.get_or_create_default_minifi_container().flow_definition.add_output_port_to_rpg(rpg_name, port_name) -@given("an output port using compression with name \"{port_name}\" is created on the RemoteProcessGroup named \"{rpg_name}\"") +@given( + 'an output port using compression with name "{port_name}" is created on the RemoteProcessGroup named "{rpg_name}"' +) def create_output_port_with_compression_on_rpg(context: MinifiTestContext, port_name: str, rpg_name: str): - context.get_or_create_default_minifi_container().flow_definition.add_output_port_to_rpg(rpg_name, port_name, use_compression=True) + context.get_or_create_default_minifi_container().flow_definition.add_output_port_to_rpg( + rpg_name, port_name, use_compression=True + ) -@given("a NiFi flow is receiving data from the RemoteProcessGroup named \"{rpg_name}\" in an input port named \"{input_port_name}\" which has the same id as the port named \"{rpg_port_name}\"") -def nifi_receive_from_rpg_input_port(context: MinifiTestContext, input_port_name: str, rpg_port_name: str, rpg_name: str): - input_port_id = context.get_or_create_default_minifi_container().flow_definition.get_input_port_id_of_rpg(rpg_name, rpg_port_name) +@given( + 'a NiFi flow is receiving data from the RemoteProcessGroup named "{rpg_name}" in an input port named "{input_port_name}" which has the same id as the port named "{rpg_port_name}"' +) +def nifi_receive_from_rpg_input_port( + context: MinifiTestContext, input_port_name: str, rpg_port_name: str, rpg_name: str +): + input_port_id = context.get_or_create_default_minifi_container().flow_definition.get_input_port_id_of_rpg( + rpg_name, rpg_port_name + ) context.containers["nifi"].flow_definition.add_input_port(input_port_id, input_port_name) -@given("a NiFi flow is sending data to an output port named \"{port_name}\" with the id of the port named \"{rpg_port_name}\" from the RemoteProcessGroup named \"{rpg_name}\"") +@given( + 'a NiFi flow is sending data to an output port named "{port_name}" with the id of the port named "{rpg_port_name}" from the RemoteProcessGroup named "{rpg_name}"' +) def nifi_send_to_rpg_output_port(context: MinifiTestContext, port_name: str, rpg_port_name: str, rpg_name: str): - output_port_id = context.get_or_create_default_minifi_container().flow_definition.get_output_port_id_of_rpg(rpg_name, rpg_port_name) + output_port_id = context.get_or_create_default_minifi_container().flow_definition.get_output_port_id_of_rpg( + rpg_name, rpg_port_name + ) context.containers["nifi"].flow_definition.add_output_port(output_port_id, port_name) -@given("the connection going to {destination} has \"drop empty\" set") +@given('the connection going to {destination} has "drop empty" set') def set_drop_empty_flag_for_connection(context: MinifiTestContext, destination: str): context.get_or_create_default_minifi_container().flow_definition.set_drop_empty_for_destination(destination) diff --git a/bootstrap/cli.py b/bootstrap/cli.py index cc821ceda1..4899a58567 100644 --- a/bootstrap/cli.py +++ b/bootstrap/cli.py @@ -15,10 +15,10 @@ import os import platform -import inquirer -import yaml from pathlib import Path +import inquirer +import yaml from minifi_option import MinifiOptions from package_manager import PackageManager from system_dependency import install_required @@ -43,7 +43,9 @@ def export_custom_conan_recipes(minifi_options: MinifiOptions, package_manager: version = next(iter(data["versions"])) print(f"Exporting the custom Conan recipe {root} with version {version}") - if not package_manager.run_cmd(f'conan export "{root}" --version={version} --user=minifi --channel=develop'): + if not package_manager.run_cmd( + f'conan export "{root}" --version={version} --user=minifi --channel=develop' + ): print(f"Exporting the custom Conan recipe {root} failed") return False return True @@ -52,19 +54,44 @@ def export_custom_conan_recipes(minifi_options: MinifiOptions, package_manager: def add_conan_options_from_cmake_options(extension_options: list[str], minifi_options: MinifiOptions) -> str: conan_options = "" for extension_option in extension_options: - if minifi_options.bool_options[extension_option.upper()].value not in (None, "OFF"): + if minifi_options.bool_options[extension_option.upper()].value not in ( + None, + "OFF", + ): conan_options += f' -o "&:{extension_option.lower()}=True"' return conan_options def run_conan_install(minifi_options: MinifiOptions, package_manager: PackageManager) -> bool: - if not minifi_options.use_conan.value == "ON": + if minifi_options.use_conan.value != "ON": print("Conan install skipped because USE_CONAN is OFF") return True - conan_options = add_conan_options_from_cmake_options(["ENABLE_ALL", "ENABLE_LIBARCHIVE", "ENABLE_ROCKSDB", "ENABLE_SFTP", "ENABLE_PROMETHEUS", "ENABLE_BZIP2", "ENABLE_LZMA", "ENABLE_MQTT", - "ENABLE_COUCHBASE", "ENABLE_KAFKA", "ENABLE_OPC", "ENABLE_GCP", "ENABLE_GRPC_FOR_LOKI", "ENABLE_BUSTACHE", "ENABLE_KUBERNETES", - "ENABLE_AZURE", "ENABLE_LLAMACPP", "ENABLE_AWS", "ENABLE_SQL", "PORTABLE", "SKIP_TESTS"], - minifi_options) + conan_options = add_conan_options_from_cmake_options( + [ + "ENABLE_ALL", + "ENABLE_LIBARCHIVE", + "ENABLE_ROCKSDB", + "ENABLE_SFTP", + "ENABLE_PROMETHEUS", + "ENABLE_BZIP2", + "ENABLE_LZMA", + "ENABLE_MQTT", + "ENABLE_COUCHBASE", + "ENABLE_KAFKA", + "ENABLE_OPC", + "ENABLE_GCP", + "ENABLE_GRPC_FOR_LOKI", + "ENABLE_BUSTACHE", + "ENABLE_KUBERNETES", + "ENABLE_AZURE", + "ENABLE_LLAMACPP", + "ENABLE_AWS", + "ENABLE_SQL", + "PORTABLE", + "SKIP_TESTS", + ], + minifi_options, + ) if minifi_options.custom_malloc is not None and minifi_options.custom_malloc.value not in (None, "OFF"): conan_options += f' -o "&:custom_malloc={minifi_options.custom_malloc.value}"' @@ -77,10 +104,10 @@ def run_conan_install(minifi_options: MinifiOptions, package_manager: PackageMan if not package_manager.run_cmd(f'cp "{settings_user}" "$(conan config home)/settings_user.yml"'): return False musl_tag_cmd = ( - 'if ldd --version 2>&1 | grep -qi musl; then ' + "if ldd --version 2>&1 | grep -qi musl; then " 'grep -q "^os.libc=" "$(conan config home)/profiles/default" || ' 'printf "\\nos.libc=musl\\n" >> "$(conan config home)/profiles/default"; ' - 'fi' + "fi" ) if not package_manager.run_cmd(musl_tag_cmd): print("Setting os.libc for musl failed") @@ -92,13 +119,19 @@ def run_conan_install(minifi_options: MinifiOptions, package_manager: PackageMan compiler_settings = " -s:a compiler.cppstd=23" if "-stdlib=libc++" in os.environ.get("CXXFLAGS", ""): compiler_settings += " -s:a compiler.libcxx=libc++" - generator_setting = " -c tools.cmake.cmaketoolchain:generator=Ninja" if minifi_options.use_ninja.value == "ON" else "" - conan_remote_add_cmd = "conan remote add nifi-conan https://apache.jfrog.io/artifactory/api/conan/nifi-conan --force" + generator_setting = ( + " -c tools.cmake.cmaketoolchain:generator=Ninja" if minifi_options.use_ninja.value == "ON" else "" + ) + conan_remote_add_cmd = ( + "conan remote add nifi-conan https://apache.jfrog.io/artifactory/api/conan/nifi-conan --force" + ) if not package_manager.run_cmd(conan_remote_add_cmd): print("Adding the nifi-conan remote failed") return False - build_cmd = f'conan install "{minifi_options.source_dir}" --output-folder="{minifi_options.build_dir}" --build=missing {conan_options} ' \ - f'--settings=build_type={minifi_options.build_type.value}{generator_setting}{compiler_settings}' + build_cmd = ( + f'conan install "{minifi_options.source_dir}" --output-folder="{minifi_options.build_dir}" --build=missing {conan_options} ' + f"--settings=build_type={minifi_options.build_type.value}{generator_setting}{compiler_settings}" + ) if platform.system() == "Linux": # Conan's prebuilt m4 binary fails autom4te's version probe in some Linux build containers (e.g. RockyLinux 8), so always build m4 from source on Linux. build_cmd += " --build=m4/*" @@ -117,27 +150,29 @@ def _conan_build_env_prefix(minifi_options: MinifiOptions) -> str: def run_cmake(minifi_options: MinifiOptions, package_manager: PackageManager): if not os.path.exists(minifi_options.build_dir): os.mkdir(minifi_options.build_dir) - cmake_cmd = f'{_conan_build_env_prefix(minifi_options)}cmake {minifi_options.create_cmake_generator_str()} {minifi_options.create_cmake_use_conan_str()} ' \ - f'{minifi_options.create_cmake_options_str()} "{minifi_options.source_dir}" -B "{minifi_options.build_dir}"' + cmake_cmd = ( + f"{_conan_build_env_prefix(minifi_options)}cmake {minifi_options.create_cmake_generator_str()} {minifi_options.create_cmake_use_conan_str()} " + f'{minifi_options.create_cmake_options_str()} "{minifi_options.source_dir}" -B "{minifi_options.build_dir}"' + ) res = package_manager.run_cmd(cmake_cmd) print("CMake command run successfully" if res else "CMake command run unsuccessfully") return res def do_build(minifi_options: MinifiOptions, package_manager: PackageManager): - build_cmd = f'{_conan_build_env_prefix(minifi_options)}cmake --build "{str(minifi_options.build_dir)}" {minifi_options.create_cmake_build_flags_str()}' + build_cmd = f'{_conan_build_env_prefix(minifi_options)}cmake --build "{minifi_options.build_dir!s}" {minifi_options.create_cmake_build_flags_str()}' res = package_manager.run_cmd(build_cmd) print("Build was successful" if res else "Build was unsuccessful") return res def do_package(minifi_options: MinifiOptions, package_manager: PackageManager): - build_cmd = f'{_conan_build_env_prefix(minifi_options)}cmake --build "{str(minifi_options.build_dir)}" --target package {minifi_options.create_cmake_build_flags_str()}' + build_cmd = f'{_conan_build_env_prefix(minifi_options)}cmake --build "{minifi_options.build_dir!s}" --target package {minifi_options.create_cmake_build_flags_str()}' return package_manager.run_cmd(build_cmd) def do_docker_build(minifi_options: MinifiOptions, package_manager: PackageManager): - build_cmd = f'cmake --build "{str(minifi_options.build_dir)}" --target docker' + build_cmd = f'cmake --build "{minifi_options.build_dir!s}" --target docker' return package_manager.run_cmd(build_cmd) @@ -229,10 +264,7 @@ def custom_malloc_menu(minifi_options: MinifiOptions, _package_manager: PackageM def build_dir_menu(minifi_options: MinifiOptions, _package_manager: PackageManager) -> bool: questions = [ - inquirer.Path('build_dir', - message="Build directory", - default=minifi_options.build_dir - ), + inquirer.Path("build_dir", message="Build directory", default=minifi_options.build_dir), ] answers = inquirer.prompt(questions) if answers is None: @@ -250,7 +282,7 @@ def extension_options_menu(minifi_options: MinifiOptions, _package_manager: Pack "options", message="MiNiFi C++ Extension Options (space to select, enter to confirm)", choices=possible_values, - default=selected_values + default=selected_values, ), ] @@ -275,7 +307,7 @@ def build_options_menu(minifi_options: MinifiOptions, _package_manager: PackageM "options", message="MiNiFi C++ Build Options (space to select, enter to confirm)", choices=possible_values, - default=selected_values + default=selected_values, ), ] @@ -318,5 +350,5 @@ def step_by_step_menu(minifi_options: MinifiOptions, package_manager: PackageMan if step_by_step_prompt is None: return True step_by_step_options[step_by_step_prompt["selection"]](minifi_options, package_manager) - done = step_by_step_prompt['selection'] == 'Back' + done = step_by_step_prompt["selection"] == "Back" return False diff --git a/bootstrap/cmake_parser.py b/bootstrap/cmake_parser.py index ff450d42d7..990547b55b 100644 --- a/bootstrap/cmake_parser.py +++ b/bootstrap/cmake_parser.py @@ -38,33 +38,37 @@ def create_cmake_option_str(self): def create_cmake_cache(cmake_path: str, cmake_options: str, directory: str, package_manager: PackageManager): - cmake_lists_path = os.path.join(directory, 'CMakeLists.txt') + cmake_lists_path = os.path.join(directory, "CMakeLists.txt") - with open(cmake_lists_path, 'w') as cmake_lists_file: - cmake_lists_file.write('cmake_minimum_required(VERSION 3.5)\n') + with open(cmake_lists_path, "w") as cmake_lists_file: + cmake_lists_file.write("cmake_minimum_required(VERSION 3.5)\n") cmake_lists_file.write(f'include("{cmake_path}")\n') if cmake_options is None: assert package_manager.run_cmd(f'cmake -G Ninja -Wno-dev --log-level=ERROR "{directory}" -B "{directory}"') else: assert package_manager.run_cmd( - f'cmake -G Ninja -Wno-dev --no-warn-unused-cli --log-level=ERROR {cmake_options} "{directory}" -B "{directory}"') - return os.path.join(directory, 'CMakeCache.txt') + f'cmake -G Ninja -Wno-dev --no-warn-unused-cli --log-level=ERROR {cmake_options} "{directory}" -B "{directory}"' + ) + return os.path.join(directory, "CMakeCache.txt") def parse_cmake_cache_values(path: str): parsed_variables = {} - with open(path, 'r') as file: + with open(path, "r") as file: contents = file.read() - pattern = r'\/\/(?P[\s\S]*?)\n(?P.+?):(?P\w+?)=(?P.+)\n' + pattern = r"\/\/(?P[\s\S]*?)\n(?P.+?):(?P\w+?)=(?P.+)\n" matches = re.findall(pattern, contents) for match in matches: - cmake_cache_value = CMakeCacheValue(description=match[0].replace("//", "").replace("\n", " "), - name=match[1], - value_type=match[2], value=match[3]) + cmake_cache_value = CMakeCacheValue( + description=match[0].replace("//", "").replace("\n", " "), + name=match[1], + value_type=match[2], + value=match[3], + ) if cmake_cache_value.name.endswith("-STRINGS"): - possible_values_of = cmake_cache_value.name[:-len("-STRINGS")] + possible_values_of = cmake_cache_value.name[: -len("-STRINGS")] if possible_values_of not in parsed_variables: raise ValueError(f"Did not parse {possible_values_of} yet") parsed_variables[possible_values_of].possible_values = cmake_cache_value.value.split(";") diff --git a/bootstrap/main.py b/bootstrap/main.py index 53be05dfab..2d70d1ac7b 100644 --- a/bootstrap/main.py +++ b/bootstrap/main.py @@ -12,31 +12,62 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import tempfile - import argparse import pathlib +import tempfile -from cli import main_menu, do_one_click_build, do_one_click_configuration, do_one_click_conan_install +from cli import ( + do_one_click_build, + do_one_click_conan_install, + do_one_click_configuration, + main_menu, +) from minifi_option import parse_minifi_options from package_manager import get_package_manager -if __name__ == '__main__': +if __name__ == "__main__": with tempfile.TemporaryDirectory() as cmake_cache_dir: parser = argparse.ArgumentParser() - parser.add_argument('--noconfirm', action="store_true", default=False, - help="Bypass any and all “Are you sure?” messages.") - parser.add_argument('--minifi-options', default="", help="Overrides the default minifi options during the " - "initial parsing") - parser.add_argument('--cmake-options', default="", help="Appends this to the final cmake command") - parser.add_argument('--skip-compiler-install', action="store_true", default=False, - help="Skips the installation of the default compiler") - parser.add_argument('--noninteractive', action="store_true", default=False, - help="Initiates the one click build") - parser.add_argument('--run-configuration', action="store_true", default=False, - help="Runs configuration") - parser.add_argument('--run-conan-install', action="store_true", default=False, - help="Runs conan install") + parser.add_argument( + "--noconfirm", + action="store_true", + default=False, + help="Bypass any and all “Are you sure?” messages.", + ) + parser.add_argument( + "--minifi-options", + default="", + help="Overrides the default minifi options during the initial parsing", + ) + parser.add_argument( + "--cmake-options", + default="", + help="Appends this to the final cmake command", + ) + parser.add_argument( + "--skip-compiler-install", + action="store_true", + default=False, + help="Skips the installation of the default compiler", + ) + parser.add_argument( + "--noninteractive", + action="store_true", + default=False, + help="Initiates the one click build", + ) + parser.add_argument( + "--run-configuration", + action="store_true", + default=False, + help="Runs configuration", + ) + parser.add_argument( + "--run-conan-install", + action="store_true", + default=False, + help="Runs conan install", + ) args = parser.parse_args() no_confirm = args.noconfirm or args.noninteractive @@ -50,12 +81,14 @@ cmake_options_for_parsing = " ".join(filter(None, [args.minifi_options, compiler_override])) cmake_options_for_cmake = " ".join(filter(None, [args.cmake_options, compiler_override])) - path = pathlib.Path(__file__).parent.resolve() / '..' / "cmake" / "MiNiFiOptions.cmake" + path = pathlib.Path(__file__).parent.resolve() / ".." / "cmake" / "MiNiFiOptions.cmake" - minifi_options = parse_minifi_options(str(path.as_posix()), - cmake_options_for_parsing, - package_manager, - cmake_cache_dir) + minifi_options = parse_minifi_options( + str(path.as_posix()), + cmake_options_for_parsing, + package_manager, + cmake_cache_dir, + ) minifi_options.load_option_state() minifi_options.no_confirm = no_confirm minifi_options.set_cmake_override(cmake_options_for_cmake) diff --git a/bootstrap/minifi_option.py b/bootstrap/minifi_option.py index 32168d2b74..70e78c6073 100644 --- a/bootstrap/minifi_option.py +++ b/bootstrap/minifi_option.py @@ -13,12 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict - -import pathlib import json -import platform import os +import pathlib +import platform import cmake_parser from cmake_parser import CMakeCacheValue @@ -26,26 +24,65 @@ class MinifiOptions: - def __init__(self, cache_values: Dict[str, CMakeCacheValue]): + def __init__(self, cache_values: dict[str, CMakeCacheValue]): self.cmake_override = "" - self.build_type = CMakeCacheValue("Specifies the build type on single-configuration generators", - "CMAKE_BUILD_TYPE", "STRING", "Release") - self.build_type.possible_values = ["Release", "Debug", "RelWithDebInfo", "MinSizeRel"] - additional_build_options = ["DOCKER_BUILD_ONLY", "DOCKER_USE_CONAN", "DOCKER_SKIP_TESTS", "DOCKER_CREATE_RPM", "SKIP_TESTS", "PORTABLE"] - self.use_ninja = CMakeCacheValue("Specifies if CMake should use the Ninja generator or the system default", "USE_NINJA", "BOOL", "ON") - self.use_conan = CMakeCacheValue("Specifies if CMake should use Conan package manager", "USE_CONAN", "BOOL", "OFF") + self.build_type = CMakeCacheValue( + "Specifies the build type on single-configuration generators", + "CMAKE_BUILD_TYPE", + "STRING", + "Release", + ) + self.build_type.possible_values = [ + "Release", + "Debug", + "RelWithDebInfo", + "MinSizeRel", + ] + additional_build_options = [ + "DOCKER_BUILD_ONLY", + "DOCKER_USE_CONAN", + "DOCKER_SKIP_TESTS", + "DOCKER_CREATE_RPM", + "SKIP_TESTS", + "PORTABLE", + ] + self.use_ninja = CMakeCacheValue( + "Specifies if CMake should use the Ninja generator or the system default", + "USE_NINJA", + "BOOL", + "ON", + ) + self.use_conan = CMakeCacheValue( + "Specifies if CMake should use Conan package manager", + "USE_CONAN", + "BOOL", + "OFF", + ) if "USE_NINJA" in cache_values: self.use_ninja.value = cache_values["USE_NINJA"].value if "USE_CONAN" in cache_values: self.use_conan.value = cache_values["USE_CONAN"].value - self.bool_options = {name: cache_value for name, cache_value in cache_values.items() if - cache_value.value_type == "BOOL" and ("ENABLE" in name or "MINIFI" in name or name in additional_build_options)} - self.build_options = {name: cache_value for name, cache_value in self.bool_options.items() if "MINIFI" in name or name in additional_build_options} + self.bool_options = { + name: cache_value + for name, cache_value in cache_values.items() + if cache_value.value_type == "BOOL" + and ("ENABLE" in name or "MINIFI" in name or name in additional_build_options) + } + self.build_options = { + name: cache_value + for name, cache_value in self.bool_options.items() + if "MINIFI" in name or name in additional_build_options + } self.build_options["USE_NINJA"] = self.use_ninja self.build_options["USE_CONAN"] = self.use_conan - self.extension_options = {name: cache_value for name, cache_value in self.bool_options.items() if "ENABLE" in name} - self.multi_choice_options = [cache_value for name, cache_value in cache_values.items() if - cache_value.value_type == "STRING" and cache_value.possible_values is not None] + self.extension_options = { + name: cache_value for name, cache_value in self.bool_options.items() if "ENABLE" in name + } + self.multi_choice_options = [ + cache_value + for name, cache_value in cache_values.items() + if cache_value.value_type == "STRING" and cache_value.possible_values is not None + ] self.custom_malloc = cache_values.get("CUSTOM_MALLOC") self.build_dir = pathlib.Path(__file__).parent.parent.resolve() / "build" self.source_dir = pathlib.Path(__file__).parent.parent.resolve() @@ -57,7 +94,7 @@ def create_cmake_options_str(self) -> str: cmake_options.append(self.custom_malloc.create_cmake_option_str()) if self.cmake_override: cmake_options.append(self.cmake_override) - cmake_options.append(f'-DCMAKE_BUILD_TYPE={self.build_type.value}') + cmake_options.append(f"-DCMAKE_BUILD_TYPE={self.build_type.value}") cmake_options_str = " ".join(filter(None, cmake_options)) return cmake_options_str @@ -86,7 +123,7 @@ def set_cmake_override(self, cmake_override: str): self.cmake_override = cmake_override def save_option_state(self): - options_dict = dict() + options_dict = {} for option_name in self.bool_options: options_dict[option_name] = self.bool_options[option_name].value options_dict[self.use_ninja.name] = self.use_ninja.value diff --git a/bootstrap/package_manager.py b/bootstrap/package_manager.py index 4f8bb0231f..07d2fb17f4 100644 --- a/bootstrap/package_manager.py +++ b/bootstrap/package_manager.py @@ -18,7 +18,6 @@ import subprocess import sys from enum import Enum -from typing import Dict, Set from distro import distro @@ -32,37 +31,41 @@ def _query_yes_no(question: str, no_confirm: bool) -> bool: valid = {"yes": True, "y": True, "ye": True, "no": False, "n": False} if no_confirm: - print("Running {} with noconfirm".format(question)) + print(f"Running {question} with noconfirm") return True while True: - print("{} [y/n]".format(question), end=' ', flush=True) + print(f"{question} [y/n]", end=" ", flush=True) choice = input().lower() if choice in valid: return valid[choice] else: - print("Please respond with 'yes' or 'no' " "(or 'y' or 'n').") + print("Please respond with 'yes' or 'no' (or 'y' or 'n').") def _run_command_with_confirm(command: str, no_confirm: bool) -> bool: - if _query_yes_no("Running {}".format(command), no_confirm): + if _query_yes_no(f"Running {command}", no_confirm): return os.system(command) == 0 -class PackageManager(object): +class PackageManager: def __init__(self, no_confirm): self.no_confirm = no_confirm - pass - def install(self, dependencies: Dict[str, Set[str]]) -> bool: - raise Exception("NotImplementedException") + def install(self, dependencies: dict[str, set[str]]) -> bool: + raise RuntimeError("NotImplementedException") def install_compiler(self) -> str: - raise Exception("NotImplementedException") + raise RuntimeError("NotImplementedException") def ensure_environment(self): pass - def _install(self, dependencies: Dict[str, Set[str]], replace_dict: Dict[str, Set[str]], install_cmd: str) -> bool: + def _install( + self, + dependencies: dict[str, set[str]], + replace_dict: dict[str, set[str]], + install_cmd: str, + ) -> bool: dependencies.update({k: v for k, v in replace_dict.items() if k in dependencies}) dependencies = self._filter_out_installed_packages(dependencies) dependencies_str = " ".join(str(value) for value_set in dependencies.values() for value in value_set) @@ -70,10 +73,10 @@ def _install(self, dependencies: Dict[str, Set[str]], replace_dict: Dict[str, Se return True return _run_command_with_confirm(f"{install_cmd} {dependencies_str}", self.no_confirm) - def _get_installed_packages(self) -> Set[str]: - raise Exception("NotImplementedException") + def _get_installed_packages(self) -> set[str]: + raise RuntimeError("NotImplementedException") - def _filter_out_installed_packages(self, dependencies: Dict[str, Set[str]]): + def _filter_out_installed_packages(self, dependencies: dict[str, set[str]]): installed_packages = self._get_installed_packages() filtered_packages = {k: (v - installed_packages) for k, v in dependencies.items()} for installed_package in installed_packages: @@ -81,7 +84,7 @@ def _filter_out_installed_packages(self, dependencies: Dict[str, Set[str]]): return filtered_packages def run_cmd(self, cmd: str) -> bool: - result = subprocess.run(f"{cmd}", shell=True, text=True) + result = subprocess.run(f"{cmd}", shell=True, text=True, check=False) return result.returncode == 0 @@ -89,24 +92,26 @@ class BrewPackageManager(PackageManager): def __init__(self, no_confirm): PackageManager.__init__(self, no_confirm) - def install(self, dependencies: Dict[str, Set[str]]) -> bool: - return self._install(dependencies=dependencies, - install_cmd="brew install", - replace_dict={"patch": set()}) + def install(self, dependencies: dict[str, set[str]]) -> bool: + return self._install( + dependencies=dependencies, + install_cmd="brew install", + replace_dict={"patch": set()}, + ) def install_compiler(self) -> str: self.install({"compiler": set()}) return "" - def _get_installed_packages(self) -> Set[str]: - result = subprocess.run(['brew', 'list'], text=True, capture_output=True, check=True) + def _get_installed_packages(self) -> set[str]: + result = subprocess.run(["brew", "list"], text=True, capture_output=True, check=True) lines = result.stdout.splitlines() - lines = [line.split('@', 1)[0] for line in lines] + lines = [line.split("@", 1)[0] for line in lines] return set(lines) def run_cmd(self, cmd: str) -> bool: add_m4_to_path_cmd = 'export PATH="$(brew --prefix m4)/bin:$PATH"' - result = subprocess.run(f"{add_m4_to_path_cmd} && {cmd}", shell=True, text=True) + result = subprocess.run(f"{add_m4_to_path_cmd} && {cmd}", shell=True, text=True, check=False) return result.returncode == 0 @@ -114,16 +119,17 @@ class AptPackageManager(PackageManager): def __init__(self, no_confirm): PackageManager.__init__(self, no_confirm) - def install(self, dependencies: Dict[str, Set[str]]) -> bool: - return self._install(dependencies=dependencies, - install_cmd="sudo apt install -y", - replace_dict={"libarchive": {"liblzma-dev"}, - "python": {"libpython3-dev"}}) - - def _get_installed_packages(self) -> Set[str]: - result = subprocess.run(['dpkg', '--get-selections'], text=True, capture_output=True, check=True) - lines = [line.split('\t')[0] for line in result.stdout.splitlines()] - lines = [line.rsplit(':', 1)[0] for line in lines] + def install(self, dependencies: dict[str, set[str]]) -> bool: + return self._install( + dependencies=dependencies, + install_cmd="sudo apt install -y", + replace_dict={"libarchive": {"liblzma-dev"}, "python": {"libpython3-dev"}}, + ) + + def _get_installed_packages(self) -> set[str]: + result = subprocess.run(["dpkg", "--get-selections"], text=True, capture_output=True, check=True) + lines = [line.split("\t")[0] for line in result.stdout.splitlines()] + lines = [line.rsplit(":", 1)[0] for line in lines] return set(lines) def install_compiler(self) -> str: @@ -136,19 +142,21 @@ def __init__(self, no_confirm, needs_epel): PackageManager.__init__(self, no_confirm) self.needs_epel = needs_epel - def install(self, dependencies: Dict[str, Set[str]]) -> bool: + def install(self, dependencies: dict[str, set[str]]) -> bool: if self.needs_epel: install_cmd = "sudo dnf --enablerepo=crb install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm" else: install_cmd = "sudo dnf install -y" - return self._install(dependencies=dependencies, - install_cmd=install_cmd, - replace_dict={"python": {"python3-devel"}}) - - def _get_installed_packages(self) -> Set[str]: - result = subprocess.run(['dnf', 'list', 'installed'], text=True, capture_output=True, check=True) - lines = [line.split(' ')[0] for line in result.stdout.splitlines()] - lines = [line.rsplit('.', 1)[0] for line in lines] + return self._install( + dependencies=dependencies, + install_cmd=install_cmd, + replace_dict={"python": {"python3-devel"}}, + ) + + def _get_installed_packages(self) -> set[str]: + result = subprocess.run(["dnf", "list", "installed"], text=True, capture_output=True, check=True) + lines = [line.split(" ")[0] for line in result.stdout.splitlines()] + lines = [line.rsplit(".", 1)[0] for line in lines] return set(lines) def install_compiler(self) -> str: @@ -160,13 +168,15 @@ class PacmanPackageManager(PackageManager): def __init__(self, no_confirm): PackageManager.__init__(self, no_confirm) - def install(self, dependencies: Dict[str, Set[str]]) -> bool: - return self._install(dependencies=dependencies, - install_cmd="sudo pacman --noconfirm -S", - replace_dict={}) + def install(self, dependencies: dict[str, set[str]]) -> bool: + return self._install( + dependencies=dependencies, + install_cmd="sudo pacman --noconfirm -S", + replace_dict={}, + ) - def _get_installed_packages(self) -> Set[str]: - result = subprocess.run(['pacman', '-Qq'], text=True, capture_output=True, check=True) + def _get_installed_packages(self) -> set[str]: + result = subprocess.run(["pacman", "-Qq"], text=True, capture_output=True, check=True) return set(result.stdout.splitlines()) def install_compiler(self) -> str: @@ -178,21 +188,30 @@ class ZypperPackageManager(PackageManager): def __init__(self, no_confirm): PackageManager.__init__(self, no_confirm) - def install(self, dependencies: Dict[str, Set[str]]) -> bool: - return self._install(dependencies=dependencies, - install_cmd="sudo zypper install -y", - replace_dict={"libarchive": {"libarchive-devel"}, - "python": {"python3-devel"}}) - - def _get_installed_packages(self) -> Set[str]: - result = subprocess.run(['zypper', 'se', '--installed-only'], text=True, capture_output=True, check=True) + def install(self, dependencies: dict[str, set[str]]) -> bool: + return self._install( + dependencies=dependencies, + install_cmd="sudo zypper install -y", + replace_dict={ + "libarchive": {"libarchive-devel"}, + "python": {"python3-devel"}, + }, + ) + + def _get_installed_packages(self) -> set[str]: + result = subprocess.run( + ["zypper", "se", "--installed-only"], + text=True, + capture_output=True, + check=True, + ) lines = result.stdout.splitlines() packages = set() for line in lines: - if line.startswith('S |') or line.startswith('--') or line.startswith(' ') or not line: + if line.startswith(("S |", "--", " ")) or not line: continue - parts = line.split('|') + parts = line.split("|") if len(parts) > 2: package_name = parts[1].strip() packages.add(package_name) @@ -208,14 +227,15 @@ class ApkPackageManager(PackageManager): def __init__(self, no_confirm): PackageManager.__init__(self, no_confirm) - def install(self, dependencies: Dict[str, Set[str]]) -> bool: - return self._install(dependencies=dependencies, - install_cmd="sudo apk add --no-cache", - replace_dict={"libarchive": {"libarchive-dev"}, - "python": {"python3-dev"}}) + def install(self, dependencies: dict[str, set[str]]) -> bool: + return self._install( + dependencies=dependencies, + install_cmd="sudo apk add --no-cache", + replace_dict={"libarchive": {"libarchive-dev"}, "python": {"python3-dev"}}, + ) - def _get_installed_packages(self) -> Set[str]: - result = subprocess.run(['apk', 'info'], text=True, capture_output=True, check=True) + def _get_installed_packages(self) -> set[str]: + result = subprocess.run(["apk", "info"], text=True, capture_output=True, check=True) return set(result.stdout.splitlines()) def install_compiler(self) -> str: @@ -234,7 +254,9 @@ def _get_vs_dev_cmd_path(vs_where_location: VsWhereLocation): f"-property installationPath " f"-requires Microsoft.VisualStudio.Component.VC.ATL " f"-version 17", - capture_output=True) + capture_output=True, + check=False, + ) for vswhere_result in vswhere_results.stdout.splitlines(): possible_path = f"{vswhere_result.decode()}\\Common7\\Tools\\VsDevCmd.bat" @@ -274,26 +296,30 @@ class ChocolateyPackageManager(PackageManager): def __init__(self, no_confirm): PackageManager.__init__(self, no_confirm) - def install(self, dependencies: Dict[str, Set[str]]) -> bool: - self._install(dependencies=dependencies, - install_cmd="choco install -y", - replace_dict={"python": set(), - "patch": set(), - "bison": set(), - "flex": set(), - "libarchive": set(), - "automake": set(), - "autoconf": set(), - "libtool": set(), - "make": set(), - "m4": {"gnuwin32-m4"}, - "perl": {"strawberryperl", "NASM"}}) + def install(self, dependencies: dict[str, set[str]]) -> bool: + self._install( + dependencies=dependencies, + install_cmd="choco install -y", + replace_dict={ + "python": set(), + "patch": set(), + "bison": set(), + "flex": set(), + "libarchive": set(), + "automake": set(), + "autoconf": set(), + "libtool": set(), + "make": set(), + "m4": {"gnuwin32-m4"}, + "perl": {"strawberryperl", "NASM"}, + }, + ) return True - def _get_installed_packages(self) -> Set[str]: - result = subprocess.run(['choco', 'list'], text=True, capture_output=True, check=True) - lines = [line.split(' ')[0] for line in result.stdout.splitlines()] - lines = [line.rsplit('.', 1)[0] for line in lines] + def _get_installed_packages(self) -> set[str]: + result = subprocess.run(["choco", "list"], text=True, capture_output=True, check=True) + lines = [line.split(" ")[0] for line in result.stdout.splitlines()] + lines = [line.rsplit(".", 1)[0] for line in lines] if os.path.exists("C:\\Program Files\\NASM"): lines.append("NASM") # choco doesnt remember NASM return set(lines) @@ -313,10 +339,17 @@ def install_compiler(self) -> str: vs_dev_path = _get_vs_dev_cmd_path(vs_where_loc) if not vs_dev_path: self.install( - {"visualstudio2022buildtools": {'visualstudio2022buildtools --package-parameters "--wait --quiet ' - '--add Microsoft.VisualStudio.Workload.VCTools ' - '--add Microsoft.VisualStudio.Component.VC.ATL ' - '--includeRecommended"'}}) + { + "visualstudio2022buildtools": { + ( + 'visualstudio2022buildtools --package-parameters "--wait --quiet ' + "--add Microsoft.VisualStudio.Workload.VCTools " + "--add Microsoft.VisualStudio.Component.VC.ATL " + '--includeRecommended"' + ) + } + } + ) return "" def run_cmd(self, cmd: str) -> bool: diff --git a/bootstrap/system_dependency.py b/bootstrap/system_dependency.py index cc97894f79..85238d14c7 100644 --- a/bootstrap/system_dependency.py +++ b/bootstrap/system_dependency.py @@ -15,25 +15,32 @@ from __future__ import annotations -from typing import Dict, Set +import platform from minifi_option import MinifiOptions from package_manager import PackageManager -import platform -def _create_system_dependencies(minifi_options: MinifiOptions) -> Dict[str, Set[str]]: - system_dependencies = {'patch': {'patch'}, 'make': {'make'}, 'perl': {'perl'}, 'git': {'git'}, 'bison': {'bison'}, 'flex': {'flex'}, 'm4': {'m4'}} +def _create_system_dependencies(minifi_options: MinifiOptions) -> dict[str, set[str]]: + system_dependencies = { + "patch": {"patch"}, + "make": {"make"}, + "perl": {"perl"}, + "git": {"git"}, + "bison": {"bison"}, + "flex": {"flex"}, + "m4": {"m4"}, + } if minifi_options.is_enabled("ENABLE_LIBARCHIVE"): - system_dependencies['libarchive'] = {'libarchive'} + system_dependencies["libarchive"] = {"libarchive"} if minifi_options.is_enabled("ENABLE_SQL"): - system_dependencies['automake'] = {'automake'} - system_dependencies['autoconf'] = {'autoconf'} - system_dependencies['libtool'] = {'libtool'} + system_dependencies["automake"] = {"automake"} + system_dependencies["autoconf"] = {"autoconf"} + system_dependencies["libtool"] = {"libtool"} if minifi_options.is_enabled("ENABLE_PYTHON_SCRIPTING"): - system_dependencies['python'] = {'python'} + system_dependencies["python"] = {"python"} if platform.system() == "Windows": - system_dependencies['wixtoolset'] = {'wixtoolset'} + system_dependencies["wixtoolset"] = {"wixtoolset"} return system_dependencies diff --git a/cmake/GenericPython.cmake b/cmake/GenericPython.cmake index a815919030..6843e3b5d0 100644 --- a/cmake/GenericPython.cmake +++ b/cmake/GenericPython.cmake @@ -15,7 +15,7 @@ # specific language governing permissions and limitations # under the License. -find_package(Python 3.6 REQUIRED COMPONENTS Development Interpreter) +find_package(Python 3.10 REQUIRED COMPONENTS Development Interpreter) if(WIN32) set(Python_LIBRARIES ${Python_LIBRARY_DIRS}/python3.lib) diff --git a/conanfile.py b/conanfile.py index f4f0f69ac2..d1c3d5be67 100644 --- a/conanfile.py +++ b/conanfile.py @@ -13,23 +13,84 @@ # See the License for the specific language governing permissions and # limitations under the License. +import os +import shutil +from typing import ClassVar + from conan import ConanFile +from conan.errors import ConanException from conan.tools.cmake import CMake, CMakeToolchain from conan.tools.files import collect_libs, copy -from conan.errors import ConanException -import os -import shutil required_conan_version = ">=2.0" -shared_requires = ("civetweb/1.16", "libxml2/2.15.3", "fmt/12.1.0", "spdlog/1.17.0", "catch2/3.15.0", "zstd/1.5.7", "sol2/3.5.0", "argparse/3.2", "libsodium/1.0.22", "gsl-lite/1.1.0", - "jsoncons/1.7.0", "json-schema-validator/2.4.0", "pugixml/1.16", "yaml-cpp/0.9.0", "range-v3/0.12.0", "magic_enum/0.9.8@minifi/develop", "date/3.0.4") +shared_requires = ( + "civetweb/1.16", + "libxml2/2.15.3", + "fmt/12.1.0", + "spdlog/1.17.0", + "catch2/3.15.0", + "zstd/1.5.7", + "sol2/3.5.0", + "argparse/3.2", + "libsodium/1.0.22", + "gsl-lite/1.1.0", + "jsoncons/1.7.0", + "json-schema-validator/2.4.0", + "pugixml/1.16", + "yaml-cpp/0.9.0", + "range-v3/0.12.0", + "magic_enum/0.9.8@minifi/develop", + "date/3.0.4", +) -shared_sources = ("CMakeLists.txt", "libminifi/*", "extensions/*", "minifi_main/*", "behave_framework/*", "bin/*", "bootstrap/*", "cmake/*", "conf/*", "controller/*", "core-framework/*", - "docs/*", "encrypt-config/*", "etc/*", "examples/*", "extension-framework/*", "fips/*", "minifi-api/*", "packaging/*", "thirdparty/*", "docker/*", "LICENSE", "NOTICE", - "README.md", "C2.md", "CONAN.md", "CONFIGURE.md", "CONTRIBUTING.md", "CONTROLLERS.md", "EXPRESSIONS.md", "Extensions.md", "METRICS.md", "OPS.md", "PARAMETER_PROVIDERS.md", - "PROCESSORS.md", "SITE_TO_SITE.md", "ThirdParties.md", "Windows.md", "CPPLINT.cfg", "generateVersion.bat", "generateVersion.sh", "run_clang_tidy.sh", "run_flake8.sh", - "run_shellcheck.sh", "versioninfo.rc.in") +shared_sources = ( + "CMakeLists.txt", + "libminifi/*", + "extensions/*", + "minifi_main/*", + "behave_framework/*", + "bin/*", + "bootstrap/*", + "cmake/*", + "conf/*", + "controller/*", + "core-framework/*", + "docs/*", + "encrypt-config/*", + "etc/*", + "examples/*", + "extension-framework/*", + "fips/*", + "minifi-api/*", + "packaging/*", + "thirdparty/*", + "docker/*", + "LICENSE", + "NOTICE", + "README.md", + "C2.md", + "CONAN.md", + "CONFIGURE.md", + "CONTRIBUTING.md", + "CONTROLLERS.md", + "EXPRESSIONS.md", + "Extensions.md", + "METRICS.md", + "OPS.md", + "PARAMETER_PROVIDERS.md", + "PROCESSORS.md", + "SITE_TO_SITE.md", + "ThirdParties.md", + "Windows.md", + "CPPLINT.cfg", + "generateVersion.bat", + "generateVersion.sh", + "run_clang_tidy.sh", + "ruff.toml", + "run_shellcheck.sh", + "versioninfo.rc.in", +) class MiNiFiCppMain(ConanFile): @@ -38,16 +99,59 @@ class MiNiFiCppMain(ConanFile): license = "Apache-2.0" settings = "os", "compiler", "build_type", "arch" generators = "CMakeDeps" - options = {"shared": [True, False], "fPIC": [True, False], "custom_malloc": [False, "jemalloc", "mimalloc", "rpmalloc"], "enable_all": [True, False], "enable_libarchive": [True, False], - "enable_rocksdb": [True, False], "enable_sftp": [True, False], "enable_prometheus": [True, False], "enable_bzip2": [True, False], "enable_lzma": [True, False], - "enable_mqtt": [True, False], "enable_couchbase": [True, False], "enable_kafka": [True, False], "enable_opc": [True, False], "enable_gcp": [True, False], - "enable_grpc_for_loki": [True, False], "enable_bustache": [True, False], "enable_kubernetes": [True, False], "enable_azure": [True, False], "enable_llamacpp": [True, False], - "enable_aws": [True, False], "enable_sql": [True, False], "skip_tests": [True, False], "portable": [True, False]} + options: ClassVar[dict] = { + "shared": [True, False], + "fPIC": [True, False], + "custom_malloc": [False, "jemalloc", "mimalloc", "rpmalloc"], + "enable_all": [True, False], + "enable_libarchive": [True, False], + "enable_rocksdb": [True, False], + "enable_sftp": [True, False], + "enable_prometheus": [True, False], + "enable_bzip2": [True, False], + "enable_lzma": [True, False], + "enable_mqtt": [True, False], + "enable_couchbase": [True, False], + "enable_kafka": [True, False], + "enable_opc": [True, False], + "enable_gcp": [True, False], + "enable_grpc_for_loki": [True, False], + "enable_bustache": [True, False], + "enable_kubernetes": [True, False], + "enable_azure": [True, False], + "enable_llamacpp": [True, False], + "enable_aws": [True, False], + "enable_sql": [True, False], + "skip_tests": [True, False], + "portable": [True, False], + } - default_options = {"shared": False, "fPIC": True, "custom_malloc": False, "enable_all": False, "enable_libarchive": False, "enable_rocksdb": False, "enable_sftp": False, - "enable_prometheus": False, "enable_bzip2": False, "enable_lzma": False, "enable_mqtt": False, "enable_couchbase": False, "enable_kafka": False, "enable_opc": False, - "enable_gcp": False, "enable_grpc_for_loki": False, "enable_bustache": False, "enable_kubernetes": False, "enable_azure": False, "enable_llamacpp": False, "enable_aws": False, - "enable_sql": False, "skip_tests": False, "portable": True} + default_options: ClassVar[dict] = { + "shared": False, + "fPIC": True, + "custom_malloc": False, + "enable_all": False, + "enable_libarchive": False, + "enable_rocksdb": False, + "enable_sftp": False, + "enable_prometheus": False, + "enable_bzip2": False, + "enable_lzma": False, + "enable_mqtt": False, + "enable_couchbase": False, + "enable_kafka": False, + "enable_opc": False, + "enable_gcp": False, + "enable_grpc_for_loki": False, + "enable_bustache": False, + "enable_kubernetes": False, + "enable_azure": False, + "enable_llamacpp": False, + "enable_aws": False, + "enable_sql": False, + "skip_tests": False, + "portable": True, + } exports_sources = shared_sources @@ -163,7 +267,7 @@ def build(self): cmake.build() def overwrite_libfile(self, oldfile, newfile): - print("Copying {} to {}".format(oldfile, newfile)) + print(f"Copying {oldfile} to {newfile}") if os.path.exists(oldfile): try: if os.path.exists(newfile): @@ -180,10 +284,34 @@ def package(self): cmake.install() include_dir = os.path.join(self.source_folder) built_dir = os.path.join(self.source_folder, self.folders.build) - copy(self, pattern="*.h*", dst=os.path.join(self.package_folder, "include"), src=include_dir, keep_path=True) - copy(self, pattern="*.i*", dst=os.path.join(self.package_folder, "include"), src=include_dir, keep_path=True) - copy(self, pattern="*.a", dst=os.path.join(self.package_folder, "lib"), src=built_dir, keep_path=False) - copy(self, pattern="*.so*", dst=os.path.join(self.package_folder, "lib"), src=built_dir, keep_path=False) + copy( + self, + pattern="*.h*", + dst=os.path.join(self.package_folder, "include"), + src=include_dir, + keep_path=True, + ) + copy( + self, + pattern="*.i*", + dst=os.path.join(self.package_folder, "include"), + src=include_dir, + keep_path=True, + ) + copy( + self, + pattern="*.a", + dst=os.path.join(self.package_folder, "lib"), + src=built_dir, + keep_path=False, + ) + copy( + self, + pattern="*.so*", + dst=os.path.join(self.package_folder, "lib"), + src=built_dir, + keep_path=False, + ) minifi_py_ext_oldfile = os.path.join(self.package_folder, "lib", "libminifi-python-script-extension.so") minifi_py_ext_copynewfile = os.path.join(self.package_folder, "lib", "libminifi_native.so") diff --git a/docker/rockylinux/Dockerfile b/docker/rockylinux/Dockerfile index 52a8362e1d..f7eb25e703 100644 --- a/docker/rockylinux/Dockerfile +++ b/docker/rockylinux/Dockerfile @@ -45,8 +45,8 @@ COPY . ${MINIFI_BASE_DIR} # ccache is in EPEL RUN dnf -y install epel-release && dnf -y install gcc-toolset-14 gcc-toolset-14-libatomic-devel sudo git which make libarchive ccache ca-certificates perl patch bison flex libtool cmake rpmdevtools && \ if echo "$MINIFI_OPTIONS" | grep -q "MINIFI_RUST=ON"; then dnf -y install rust cargo clang; fi && \ - if [ "${DOCKER_USE_CONAN}" == "ON" ]; then dnf -y install python3.9; fi && \ - if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_ALL=ON\|ENABLE_PYTHON_SCRIPTING=ON\|ENABLE_OPC=ON"; then dnf -y --enablerepo=devel install python3-devel; fi && \ + if [ "${DOCKER_USE_CONAN}" == "ON" ]; then dnf -y install python3.12; fi && \ + if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_ALL=ON\|ENABLE_PYTHON_SCRIPTING=ON\|ENABLE_OPC=ON"; then dnf -y --enablerepo=devel install python3.12-devel; fi && \ if echo "$MINIFI_OPTIONS" | grep -q "ENABLE_ALL=ON\|ENABLE_SFTP=ON" && [ "${DOCKER_SKIP_TESTS}" == "OFF" ]; then dnf -y install java-1.8.0-openjdk maven; fi RUN cd $MINIFI_BASE_DIR && \ @@ -61,11 +61,11 @@ USER ${USER} RUN --mount=type=secret,id=nifi_conan_password,mode=0444 echo "DOCKER_USE_CONAN=${DOCKER_USE_CONAN}" && if [ "${DOCKER_USE_CONAN}" == "ON" ]; then \ set -e; \ cd ${MINIFI_BASE_DIR}/bootstrap && \ - python3.9 -m venv venv && \ + python3.12 -m venv venv && \ source /opt/rh/gcc-toolset-14/enable && \ source venv/bin/activate && \ - pip3.9 install -r requirements.txt && \ - python3.9 main.py --skip-compiler-install --run-conan-install --minifi-options="${MINIFI_OPTIONS}" && \ + pip3.12 install -r requirements.txt && \ + python3.12 main.py --skip-compiler-install --run-conan-install --minifi-options="${MINIFI_OPTIONS}" && \ if [ -n "${DOCKER_NIFI_CONAN_USER}" ] && [ -f /run/secrets/nifi_conan_password ]; then \ conan remote login nifi-conan ${DOCKER_NIFI_CONAN_USER} -p "$(cat /run/secrets/nifi_conan_password)" || true; \ conan upload "*" -r nifi-conan --confirm || true; \ @@ -80,7 +80,7 @@ RUN cd $MINIFI_BASE_DIR && \ export PATH=/usr/lib64/ccache${PATH:+:${PATH}} && \ export CCACHE_DIR=${MINIFI_BASE_DIR}/.ccache && \ if [ "${DOCKER_USE_CONAN}" == "ON" ]; then CONAN_TOOLCHAIN="-DCMAKE_TOOLCHAIN_FILE=conan_toolchain.cmake"; fi && \ - cmake -DSTATIC_BUILD= -DSKIP_TESTS=${DOCKER_SKIP_TESTS} ${CONAN_TOOLCHAIN:-} ${MINIFI_OPTIONS} -DPython_EXECUTABLE=/usr/bin/python3.6 -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. && \ + cmake -DSTATIC_BUILD= -DSKIP_TESTS=${DOCKER_SKIP_TESTS} ${CONAN_TOOLCHAIN:-} ${MINIFI_OPTIONS} -DPython_EXECUTABLE=/usr/bin/python3.12 -DAWS_ENABLE_UNITY_BUILD=OFF -DCMAKE_BUILD_TYPE="${CMAKE_BUILD_TYPE}" .. && \ make -j "$(nproc)" ${DOCKER_MAKE_TARGET} && \ if [[ "${DOCKER_CREATE_RPM}" == "ON" ]] ; then \ cmake -DMINIFI_PACKAGING_TYPE=RPM .. && \ diff --git a/examples/scripts/python/reverse_flow_file_content.py b/examples/scripts/python/reverse_flow_file_content.py index 66e76c0ba9..da9a87c55b 100644 --- a/examples/scripts/python/reverse_flow_file_content.py +++ b/examples/scripts/python/reverse_flow_file_content.py @@ -21,7 +21,7 @@ class ReadCallback: def process(self, input_stream): - self.content = codecs.getreader('utf-8')(input_stream).read() + self.content = codecs.getreader("utf-8")(input_stream).read() return len(self.content) @@ -31,7 +31,7 @@ def __init__(self, content): def process(self, output_stream): reversed_content = self.content[::-1] - output_stream.write(reversed_content.encode('utf-8')) + output_stream.write(reversed_content.encode("utf-8")) return len(reversed_content) @@ -41,5 +41,5 @@ def onTrigger(context, session): read_callback = ReadCallback() session.read(flow_file, read_callback) session.write(flow_file, WriteReverseStringCallback(read_callback.content)) - flow_file.addAttribute('python_timestamp', str(int(time.time()))) + flow_file.addAttribute("python_timestamp", str(int(time.time()))) session.transfer(flow_file, REL_SUCCESS) diff --git a/extensions/aws/tests/features/containers/kinesis_server_container.py b/extensions/aws/tests/features/containers/kinesis_server_container.py index f7767773c1..e786a05452 100644 --- a/extensions/aws/tests/features/containers/kinesis_server_container.py +++ b/extensions/aws/tests/features/containers/kinesis_server_container.py @@ -14,23 +14,29 @@ # limitations under the License. import logging - from pathlib import Path + from minifi_behave.containers.container_linux import LinuxContainer -from minifi_behave.core.helpers import wait_for_condition, retry_check -from minifi_behave.core.minifi_test_context import MinifiTestContext from minifi_behave.containers.docker_image_builder import DockerImageBuilder +from minifi_behave.core.helpers import retry_check, wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext + +logger = logging.getLogger(__name__) class KinesisServerContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): builder = DockerImageBuilder( image_tag="minifi-kinesis-mock:latest", - build_context_path=str(Path(__file__).resolve().parent.parent / "resources" / "kinesis-mock") + build_context_path=str(Path(__file__).resolve().parent.parent / "resources" / "kinesis-mock"), ) builder.build() - super().__init__("minifi-kinesis-mock:latest", f"kinesis-server-{test_context.scenario_id}", test_context.network) + super().__init__( + "minifi-kinesis-mock:latest", + f"kinesis-server-{test_context.scenario_id}", + test_context.network, + ) self.environment.append("INITIALIZE_STREAMS=test_stream:3") self.environment.append("LOG_LEVEL=DEBUG") @@ -41,10 +47,11 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=300, bail_condition=lambda: self.exited, - context=context) + context=context, + ) @retry_check() def check_kinesis_server_record_data(self, record_data): (code, output) = self.exec_run(["node", "/app/consumer/consumer.js", record_data]) - logging.info(f"Kinesis server returned output: '{output}' with code '{code}'") + logger.info(f"Kinesis server returned output: '{output}' with code '{code}'") return code == 0 diff --git a/extensions/aws/tests/features/containers/s3_server_container.py b/extensions/aws/tests/features/containers/s3_server_container.py index b2b76ffb12..3fde692d3b 100644 --- a/extensions/aws/tests/features/containers/s3_server_container.py +++ b/extensions/aws/tests/features/containers/s3_server_container.py @@ -20,12 +20,18 @@ from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +logger = logging.getLogger(__name__) + class S3ServerContainer(LinuxContainer): IMAGE = "adobe/s3mock:3.12.0" def __init__(self, test_context: MinifiTestContext): - super().__init__(S3ServerContainer.IMAGE, f"s3-server-{test_context.scenario_id}", test_context.network) + super().__init__( + S3ServerContainer.IMAGE, + f"s3-server-{test_context.scenario_id}", + test_context.network, + ) self.environment.append("initialBuckets=test_bucket") def deploy(self, context: MinifiTestContext | None) -> bool: @@ -35,10 +41,22 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=60, bail_condition=lambda: self.exited, - context=context) + context=context, + ) def check_s3_server_object_data(self, test_data): - (code, output) = self.exec_run(["find", "/s3mockroot/test_bucket", "-mindepth", "1", "-maxdepth", "1", "-type", "d"]) + (code, output) = self.exec_run( + [ + "find", + "/s3mockroot/test_bucket", + "-mindepth", + "1", + "-maxdepth", + "1", + "-type", + "d", + ] + ) if code != 0: return False s3_mock_dir = output.strip() @@ -46,7 +64,18 @@ def check_s3_server_object_data(self, test_data): return code == 0 and file_data == test_data def check_s3_server_object_hash(self, expected_file_hash: str): - (code, output) = self.exec_run(["find", "/s3mockroot/test_bucket", "-mindepth", "1", "-maxdepth", "1", "-type", "d"]) + (code, output) = self.exec_run( + [ + "find", + "/s3mockroot/test_bucket", + "-mindepth", + "1", + "-maxdepth", + "1", + "-type", + "d", + ] + ) if code != 0: return False dir_candidates = output.split("\n") @@ -57,19 +86,45 @@ def check_s3_server_object_hash(self, expected_file_hash: str): (code, md5_output) = self.exec_run(["md5sum", s3_mock_dir + "/binaryData"]) if code != 0: return False - file_hash = md5_output.split(' ')[0].strip() + file_hash = md5_output.split(" ")[0].strip() return file_hash == expected_file_hash - def check_s3_server_object_metadata(self, content_type="application/octet-stream", metadata=dict()): - (code, output) = self.exec_run(["find", "/s3mockroot/test_bucket", "-mindepth", "1", "-maxdepth", "1", "-type", "d"]) + def check_s3_server_object_metadata(self, content_type="application/octet-stream", metadata=None): + if metadata is None: + metadata = {} + (code, output) = self.exec_run( + [ + "find", + "/s3mockroot/test_bucket", + "-mindepth", + "1", + "-maxdepth", + "1", + "-type", + "d", + ] + ) if code != 0: return False s3_mock_dir = output.strip() (code, output) = self.exec_run(["cat", s3_mock_dir + "/objectMetadata.json"]) server_metadata = json.loads(output) - return code == 0 and server_metadata["contentType"] == content_type and metadata == server_metadata["userMetadata"] + return ( + code == 0 and server_metadata["contentType"] == content_type and metadata == server_metadata["userMetadata"] + ) def is_s3_bucket_empty(self): - (code, output) = self.exec_run(["find", "/s3mockroot/test_bucket", "-mindepth", "1", "-maxdepth", "1", "-type", "d"]) - logging.info(f"is_s3_bucket_empty: {output}") + (code, output) = self.exec_run( + [ + "find", + "/s3mockroot/test_bucket", + "-mindepth", + "1", + "-maxdepth", + "1", + "-type", + "d", + ] + ) + logger.info(f"is_s3_bucket_empty: {output}") return code == 0 and not output.strip() diff --git a/extensions/aws/tests/features/environment.py b/extensions/aws/tests/features/environment.py index 08e04da68f..b68ea0e23f 100644 --- a/extensions/aws/tests/features/environment.py +++ b/extensions/aws/tests/features/environment.py @@ -13,15 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario from containers.s3_server_container import S3ServerContainer +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario +import docker # These hooks are executed by behave before and after each scenario # The common_before_scenario and common_after_scenario must be called for proper setup and tear down + def before_feature(context, feature): if feature.filename.endswith("s3.feature"): docker.from_env().images.pull(S3ServerContainer.IMAGE) diff --git a/extensions/aws/tests/features/steps/steps.py b/extensions/aws/tests/features/steps/steps.py index ab10940464..d190f1e331 100644 --- a/extensions/aws/tests/features/steps/steps.py +++ b/extensions/aws/tests/features/steps/steps.py @@ -19,38 +19,38 @@ import humanfriendly from behave import step, then - +from containers.kinesis_server_container import KinesisServerContainer +from containers.s3_server_container import S3ServerContainer from minifi_behave.containers.directory import Directory -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from minifi_behave.core.helpers import log_due_to_failure, wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext from minifi_behave.minifi.processor import Processor -from minifi_behave.core.helpers import wait_for_condition, log_due_to_failure - -from containers.s3_server_container import S3ServerContainer -from containers.kinesis_server_container import KinesisServerContainer +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) -@step('a {processor_name} processor set up to communicate with an s3 server') -@step('a {processor_name} processor set up to communicate with the same s3 server') +@step("a {processor_name} processor set up to communicate with an s3 server") +@step("a {processor_name} processor set up to communicate with the same s3 server") def setup_s3_processor(context: MinifiTestContext, processor_name: str): processor = Processor(processor_name, processor_name) - processor.add_property('Object Key', 'test_object_key') - processor.add_property('Bucket', 'test_bucket') - processor.add_property('Access Key', 'test_access_key') - processor.add_property('Secret Key', 'test_secret') - processor.add_property('Endpoint Override URL', f"http://s3-server-{context.scenario_id}:9090") - processor.add_property('Proxy Host', '') - processor.add_property('Proxy Port', '') - processor.add_property('Proxy Username', '') - processor.add_property('Proxy Password', '') + processor.add_property("Object Key", "test_object_key") + processor.add_property("Bucket", "test_bucket") + processor.add_property("Access Key", "test_access_key") + processor.add_property("Secret Key", "test_secret") + processor.add_property("Endpoint Override URL", f"http://s3-server-{context.scenario_id}:9090") + processor.add_property("Proxy Host", "") + processor.add_property("Proxy Port", "") + processor.add_property("Proxy Username", "") + processor.add_property("Proxy Password", "") context.get_or_create_default_minifi_container().flow_definition.add_processor(processor) -@step('the s3 server starts up') +@step("the s3 server starts up") def start_s3_server(context: MinifiTestContext): context.containers["s3-server"] = S3ServerContainer(context) assert context.containers["s3-server"].deploy(context) @@ -76,7 +76,10 @@ def verify_s3_bucket_empty(context): assert isinstance(s3_server_container, S3ServerContainer) assert wait_for_condition( condition=lambda: s3_server_container.is_s3_bucket_empty(), - timeout_seconds=10, bail_condition=lambda: s3_server_container.exited, context=context) + timeout_seconds=10, + bail_condition=lambda: s3_server_container.exited, + context=context, + ) @step("the object on the s3 server is present and matches the original hash") @@ -89,14 +92,14 @@ def verify_s3_object_hash_matches(context): def computeMD5hash(my_string): m = hashlib.md5() - m.update(my_string.encode('utf-8')) + m.update(my_string.encode("utf-8")) return m.hexdigest() @step('there is a 6MB file at the "/tmp/input" directory and we keep track of the hash of that') def create_6mb_file_and_track_hash(context): size = humanfriendly.parse_size("6MB") - content = ''.join(random.choice(string.ascii_uppercase + string.digits) for _ in range(size)) + content = "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(size)) new_dir = Directory("/tmp/input") new_dir.files["input.txt"] = content context.get_or_create_default_minifi_container().dirs.append(new_dir) @@ -108,6 +111,8 @@ def setup_kinesis_server(context): context.containers["kinesis-server"] = KinesisServerContainer(context) -@then("there is a record on the kinesis server with \"{record_data}\"") +@then('there is a record on the kinesis server with "{record_data}"') def verify_kinesis_record_data(context, record_data): - assert context.containers["kinesis-server"].check_kinesis_server_record_data(record_data) or log_due_to_failure(context) + assert context.containers["kinesis-server"].check_kinesis_server_record_data(record_data) or log_due_to_failure( + context + ) diff --git a/extensions/azure/tests/features/containers/azure_server_container.py b/extensions/azure/tests/features/containers/azure_server_container.py index 7d99ae912e..c1a5a0b521 100644 --- a/extensions/azure/tests/features/containers/azure_server_container.py +++ b/extensions/azure/tests/features/containers/azure_server_container.py @@ -19,30 +19,48 @@ from docker.errors import ContainerError from minifi_behave.containers.container_linux import LinuxContainer -from minifi_behave.core.helpers import run_cmd_in_docker_image -from minifi_behave.core.helpers import wait_for_condition +from minifi_behave.core.helpers import run_cmd_in_docker_image, wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +logger = logging.getLogger(__name__) + class AzureServerContainer(LinuxContainer): IMAGE = "mcr.microsoft.com/azure-storage/azurite:3.35.0" def __init__(self, test_context: MinifiTestContext): - super().__init__(AzureServerContainer.IMAGE, - f"azure-storage-server-{test_context.scenario_id}", - test_context.network) + super().__init__( + AzureServerContainer.IMAGE, + f"azure-storage-server-{test_context.scenario_id}", + test_context.network, + ) azure_storage_hostname = f"azure-storage-server-{test_context.scenario_id}" - self.azure_connection_string = "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" \ - f"BlobEndpoint=http://{azure_storage_hostname}:10000/devstoreaccount1;QueueEndpoint=http://{azure_storage_hostname}:10001/devstoreaccount1;" - self.command = ["azurite", "-l", "/data", "--blobHost", "0.0.0.0", "--queueHost", "0.0.0.0", "--tableHost", "0.0.0.0", "--skipApiVersionCheck"] + self.azure_connection_string = ( + "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;" + f"BlobEndpoint=http://{azure_storage_hostname}:10000/devstoreaccount1;QueueEndpoint=http://{azure_storage_hostname}:10001/devstoreaccount1;" + ) + self.command = [ + "azurite", + "-l", + "/data", + "--blobHost", + "0.0.0.0", + "--queueHost", + "0.0.0.0", + "--tableHost", + "0.0.0.0", + "--skipApiVersionCheck", + ] def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) finished_str = "Azurite Queue service is successfully listening at" - return wait_for_condition(condition=lambda: finished_str in self.get_logs(), - timeout_seconds=15, - bail_condition=lambda: self.exited, - context=context) + return wait_for_condition( + condition=lambda: finished_str in self.get_logs(), + timeout_seconds=15, + bail_condition=lambda: self.exited, + context=context, + ) def check_azure_storage_server_data(self, test_data): (code, output) = self.exec_run(["find", "/data/__blobstorage__", "-type", "f"]) @@ -53,48 +71,84 @@ def check_azure_storage_server_data(self, test_data): return code == 0 and test_data in file_data def add_test_blob(self, blob_name, content="test_data", with_snapshot=False) -> bool: - cmd_create = ["az", "storage", "container", "create", "--name", "test-container", "--connection-string", - self.azure_connection_string] + cmd_create = [ + "az", + "storage", + "container", + "create", + "--name", + "test-container", + "--connection-string", + self.azure_connection_string, + ] try: run_cmd_in_docker_image("mcr.microsoft.com/azure-cli:2.81.0", cmd_create, self.network.name) except ContainerError as e: - logging.error(e) + logger.error(e) return False - cmd_upload = ["az", "storage", "blob", "upload", "--container-name", "test-container", "--name", blob_name, - "--data", content, "--connection-string", self.azure_connection_string] + cmd_upload = [ + "az", + "storage", + "blob", + "upload", + "--container-name", + "test-container", + "--name", + blob_name, + "--data", + content, + "--connection-string", + self.azure_connection_string, + ] try: run_cmd_in_docker_image("mcr.microsoft.com/azure-cli:2.81.0", cmd_upload, self.network.name) except ContainerError as e: - logging.error(e) + logger.error(e) return False if with_snapshot: - cmd_snapshot = ["az", "storage", "blob", "snapshot", "--container-name", "test-container", "--name", - blob_name, "--connection-string", self.azure_connection_string] + cmd_snapshot = [ + "az", + "storage", + "blob", + "snapshot", + "--container-name", + "test-container", + "--name", + blob_name, + "--connection-string", + self.azure_connection_string, + ] try: - run_cmd_in_docker_image("mcr.microsoft.com/azure-cli:2.81.0", cmd_snapshot, self.network.name) + run_cmd_in_docker_image( + "mcr.microsoft.com/azure-cli:2.81.0", + cmd_snapshot, + self.network.name, + ) except ContainerError as e: - logging.error(e) + logger.error(e) return False return True def __get_blob_and_snapshot_count(self) -> int: - cmd = (f'az storage blob list --container-name "test-container" ' - f'--include deleted --query "length(@)" --output tsv ' - f'--connection-string "{self.azure_connection_string}"') + cmd = ( + f'az storage blob list --container-name "test-container" ' + f'--include deleted --query "length(@)" --output tsv ' + f'--connection-string "{self.azure_connection_string}"' + ) try: output = run_cmd_in_docker_image("mcr.microsoft.com/azure-cli:2.81.0", cmd, self.network.name) except ContainerError as e: - logging.error(e) + logger.error(e) return -1 try: return int(output.strip()) except (ValueError, TypeError): - logging.error(f"{output} Not an int") + logger.error(f"{output} Not an int") return -1 def check_azure_blob_and_snapshot_count(self, blob_and_snapshot_count) -> bool: diff --git a/extensions/azure/tests/features/environment.py b/extensions/azure/tests/features/environment.py index 4684766146..64e98934e9 100644 --- a/extensions/azure/tests/features/environment.py +++ b/extensions/azure/tests/features/environment.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario from containers.azure_server_container import AzureServerContainer +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker # These hooks are executed by behave before and after each scenario # The common_before_scenario and common_after_scenario must be called for proper setup and tear down diff --git a/extensions/azure/tests/features/steps/steps.py b/extensions/azure/tests/features/steps/steps.py index 25409d4275..03d08d8031 100644 --- a/extensions/azure/tests/features/steps/steps.py +++ b/extensions/azure/tests/features/steps/steps.py @@ -17,26 +17,29 @@ import humanfriendly from behave import step, then +from containers.azure_server_container import AzureServerContainer from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext from minifi_behave.minifi.processor import Processor -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 - -from containers.azure_server_container import AzureServerContainer +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) @step("a {processor_name} processor set up to communicate with an Azure blob storage") def setup_azure_blob_storage_processor(context: MinifiTestContext, processor_name: str): processor = Processor(processor_name, processor_name) hostname = f"http://azure-storage-server-{context.scenario_id}" - processor.add_property('Container Name', 'test-container') - processor.add_property('Connection String', - f'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={hostname}:10000/devstoreaccount1;QueueEndpoint={hostname}:10001/devstoreaccount1;') - processor.add_property('Blob', 'test-blob') - processor.add_property('Create Container', 'true') + processor.add_property("Container Name", "test-container") + processor.add_property( + "Connection String", + f"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint={hostname}:10000/devstoreaccount1;QueueEndpoint={hostname}:10001/devstoreaccount1;", + ) + processor.add_property("Blob", "test-blob") + processor.add_property("Create Container", "true") context.get_or_create_default_minifi_container().flow_definition.add_processor(processor) @@ -83,7 +86,8 @@ def verify_azure_blob_storage_is_empty(context: MinifiTestContext, timeout_str: condition=lambda: azure_server_container.check_azure_blob_storage_is_empty(), timeout_seconds=timeout_in_seconds, bail_condition=lambda: azure_server_container.exited, - context=context) + context=context, + ) @then("the blob and snapshot count becomes 1 in {timeout_str}") @@ -95,4 +99,5 @@ def verify_blob_and_snapshot_count(context: MinifiTestContext, timeout_str: str) condition=lambda: azure_server_container.check_azure_blob_and_snapshot_count(1), timeout_seconds=timeout_in_seconds, bail_condition=lambda: azure_server_container.exited, - context=context) + context=context, + ) diff --git a/extensions/civetweb/tests/features/environment.py b/extensions/civetweb/tests/features/environment.py index 5b3659f585..5c865091c3 100644 --- a/extensions/civetweb/tests/features/environment.py +++ b/extensions/civetweb/tests/features/environment.py @@ -13,8 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario def before_scenario(context, scenario): diff --git a/extensions/civetweb/tests/features/steps/steps.py b/extensions/civetweb/tests/features/steps/steps.py index 25759880fa..ff5817598b 100644 --- a/extensions/civetweb/tests/features/steps/steps.py +++ b/extensions/civetweb/tests/features/steps/steps.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) diff --git a/extensions/couchbase/tests/features/containers/couchbase_server_container.py b/extensions/couchbase/tests/features/containers/couchbase_server_container.py index 41c8a89eed..61dc7e7f15 100644 --- a/extensions/couchbase/tests/features/containers/couchbase_server_container.py +++ b/extensions/couchbase/tests/features/containers/couchbase_server_container.py @@ -15,28 +15,54 @@ import logging -from minifi_behave.core.helpers import wait_for_condition, retry_check -from minifi_behave.core.ssl_utils import make_server_cert, dump_cert, dump_key +from docker.errors import ContainerError from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.containers.file import File +from minifi_behave.core.helpers import retry_check, wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext -from docker.errors import ContainerError +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert + +logger = logging.getLogger(__name__) class CouchbaseServerContainer(LinuxContainer): IMAGE = "couchbase:enterprise-7.2.5" def __init__(self, test_context: MinifiTestContext): - super().__init__(CouchbaseServerContainer.IMAGE, f"couchbase-server-{test_context.scenario_id}", test_context.network) + super().__init__( + CouchbaseServerContainer.IMAGE, + f"couchbase-server-{test_context.scenario_id}", + test_context.network, + ) - couchbase_cert, couchbase_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + couchbase_cert, couchbase_key = make_server_cert( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) root_ca_content = dump_cert(test_context.root_ca_cert) - self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt", root_ca_content, permissions=0o666)) + self.files.append( + File( + "/opt/couchbase/var/lib/couchbase/inbox/CA/root_ca.crt", + root_ca_content, + permissions=0o666, + ) + ) couchbase_cert_content = dump_cert(couchbase_cert) - self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/chain.pem", couchbase_cert_content, permissions=0o666)) + self.files.append( + File( + "/opt/couchbase/var/lib/couchbase/inbox/chain.pem", + couchbase_cert_content, + permissions=0o666, + ) + ) couchbase_key_content = dump_key(couchbase_key) - self.files.append(File("/opt/couchbase/var/lib/couchbase/inbox/pkey.key", couchbase_key_content, permissions=0o666)) + self.files.append( + File( + "/opt/couchbase/var/lib/couchbase/inbox/pkey.key", + couchbase_key_content, + permissions=0o666, + ) + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -45,33 +71,93 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=30, bail_condition=lambda: self.exited, - context=context) + context=context, + ) return self.run_post_startup_commands() def run_post_startup_commands(self): commands = [ - ["couchbase-cli", "cluster-init", "-c", "localhost", "--cluster-username", "Administrator", "--cluster-password", "password123", "--services", "data,index,query", - "--cluster-ramsize", "2048", "--cluster-index-ramsize", "256"], - ["couchbase-cli", "bucket-create", "-c", "localhost", "--username", "Administrator", "--password", "password123", "--bucket", "test_bucket", "--bucket-type", "couchbase", - "--bucket-ramsize", "1024", "--max-ttl", "36000"], - ["couchbase-cli", "user-manage", "-c", "localhost", "-u", "Administrator", "-p", "password123", "--set", "--rbac-username", "clientuser", "--rbac-password", "password123", - "--roles", "data_reader[test_bucket],data_writer[test_bucket]", "--auth-domain", "local"], - ["bash", "-c", 'tee /tmp/auth.json <<< \'{"state": "enable", "prefixes": [ {"path": "subject.cn", "prefix": "", "delimiter": "."}]}\''], - ['couchbase-cli', 'ssl-manage', '-c', 'localhost', '-u', 'Administrator', '-p', 'password123', '--set-client-auth', '/tmp/auth.json'] + [ + "couchbase-cli", + "cluster-init", + "-c", + "localhost", + "--cluster-username", + "Administrator", + "--cluster-password", + "password123", + "--services", + "data,index,query", + "--cluster-ramsize", + "2048", + "--cluster-index-ramsize", + "256", + ], + [ + "couchbase-cli", + "bucket-create", + "-c", + "localhost", + "--username", + "Administrator", + "--password", + "password123", + "--bucket", + "test_bucket", + "--bucket-type", + "couchbase", + "--bucket-ramsize", + "1024", + "--max-ttl", + "36000", + ], + [ + "couchbase-cli", + "user-manage", + "-c", + "localhost", + "-u", + "Administrator", + "-p", + "password123", + "--set", + "--rbac-username", + "clientuser", + "--rbac-password", + "password123", + "--roles", + "data_reader[test_bucket],data_writer[test_bucket]", + "--auth-domain", + "local", + ], + [ + "bash", + "-c", + 'tee /tmp/auth.json <<< \'{"state": "enable", "prefixes": [ {"path": "subject.cn", "prefix": "", "delimiter": "."}]}\'', + ], + [ + "couchbase-cli", + "ssl-manage", + "-c", + "localhost", + "-u", + "Administrator", + "-p", + "password123", + "--set-client-auth", + "/tmp/auth.json", + ], ] if not self._run_couchbase_cli_commands(commands): return False - if not self._load_couchbase_certs(): - return False - - return True + return self._load_couchbase_certs() @retry_check(max_tries=12, retry_interval_seconds=5) def _run_couchbase_cli_command(self, command): (code, output) = self.exec_run(command) if code != 0: - logging.error(f"Failed to run command '{command}', returned error code: {code}, output: '{output}'") + logger.error(f"Failed to run command '{command}', returned error code: {code}, output: '{output}'") return False return True @@ -80,15 +166,24 @@ def _run_couchbase_cli_commands(self, commands): def _run_python_in_couchbase_helper_docker(self, command: str): try: - self.client.containers.run("minifi-couchbase-helper:latest", ["python", "-c", command], remove=True, stdout=True, stderr=True, network=self.network.name) + self.client.containers.run( + "minifi-couchbase-helper:latest", + ["python", "-c", command], + remove=True, + stdout=True, + stderr=True, + network=self.network.name, + ) return True except ContainerError as e: stdout = e.stdout.decode("utf-8", errors="replace") if hasattr(e, "stdout") and e.stdout else "" stderr = e.stderr.decode("utf-8", errors="replace") if hasattr(e, "stderr") and e.stderr else "" - logging.error(f"Python command '{command}' failed in couchbase helper docker with error: '{e}', stdout: '{stdout}', stderr: '{stderr}'") + logger.error( + f"Python command '{command}' failed in couchbase helper docker with error: '{e}', stdout: '{stdout}', stderr: '{stderr}'" + ) return False except Exception as e: - logging.error(f"Unexpected error while running python command '{command}' in couchbase helper docker: '{e}'") + logger.error(f"Unexpected error while running python command '{command}' in couchbase helper docker: '{e}'") return False @retry_check(max_tries=15, retry_interval_seconds=2) diff --git a/extensions/couchbase/tests/features/environment.py b/extensions/couchbase/tests/features/environment.py index d48dc49e15..02c579f4f8 100644 --- a/extensions/couchbase/tests/features/environment.py +++ b/extensions/couchbase/tests/features/environment.py @@ -13,21 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker -from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario from containers.couchbase_server_container import CouchbaseServerContainer +from minifi_behave.containers.docker_image_builder import DockerImageBuilder +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_all(context): dockerfile = """ FROM python:3.13-slim-bookworm RUN pip install couchbase==4.3.5 requests""" - builder = DockerImageBuilder( - image_tag="minifi-couchbase-helper:latest", - dockerfile_content=dockerfile - ) + builder = DockerImageBuilder(image_tag="minifi-couchbase-helper:latest", dockerfile_content=dockerfile) builder.build() docker.from_env().images.pull(CouchbaseServerContainer.IMAGE) diff --git a/extensions/couchbase/tests/features/steps/steps.py b/extensions/couchbase/tests/features/steps/steps.py index 9f0edeffc3..996d923830 100644 --- a/extensions/couchbase/tests/features/steps/steps.py +++ b/extensions/couchbase/tests/features/steps/steps.py @@ -13,16 +13,17 @@ # See the License for the specific language governing permissions and # limitations under the License. from behave import step, then - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.steps.flow_building_steps import add_ssl_context_service_for_minifi -from minifi_behave.core.minifi_test_context import MinifiTestContext +from containers.couchbase_server_container import CouchbaseServerContainer from minifi_behave.core.helpers import log_due_to_failure +from minifi_behave.core.minifi_test_context import MinifiTestContext from minifi_behave.minifi.controller_service import ControllerService -from containers.couchbase_server_container import CouchbaseServerContainer +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) +from minifi_behave.steps.flow_building_steps import add_ssl_context_service_for_minifi @step("a Couchbase server is started") @@ -45,12 +46,16 @@ def setup_couchbase_cluster_service_ssl(context): ssl_context_service = ControllerService(class_name="SSLContextService", service_name="SSLContextService") ssl_context_service.add_property("CA Certificate", "/tmp/resources/root_ca.crt") context.get_or_create_default_minifi_container().flow_definition.controller_services.append(ssl_context_service) - couchbase_cluster_service = ControllerService(class_name="CouchbaseClusterService", service_name="CouchbaseClusterService") + couchbase_cluster_service = ControllerService( + class_name="CouchbaseClusterService", service_name="CouchbaseClusterService" + ) couchbase_cluster_service.add_property("Connection String", f"couchbases://couchbase-server-{context.scenario_id}") couchbase_cluster_service.add_property("User Name", "Administrator") couchbase_cluster_service.add_property("User Password", "password123") couchbase_cluster_service.add_property("Linked Services", "SSLContextService") - context.get_or_create_default_minifi_container().flow_definition.controller_services.append(couchbase_cluster_service) + context.get_or_create_default_minifi_container().flow_definition.controller_services.append( + couchbase_cluster_service + ) @step("a CouchbaseClusterService is setup up using mTLS authentication") @@ -62,6 +67,10 @@ def setup_couchbase_cluster_service_mtls(context: MinifiTestContext): context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) -@then("a document with id \"{doc_id}\" in bucket \"{bucket_name}\" is present with data '{data}' of type \"{data_type}\" in Couchbase") +@then( + 'a document with id "{doc_id}" in bucket "{bucket_name}" is present with data \'{data}\' of type "{data_type}" in Couchbase' +) def verify_couchbase_document_data(context, doc_id: str, bucket_name: str, data: str, data_type: str): - assert context.containers["couchbase-server"].is_data_present_in_couchbase(doc_id, bucket_name, data, data_type) or log_due_to_failure(context) + assert context.containers["couchbase-server"].is_data_present_in_couchbase( + doc_id, bucket_name, data, data_type + ) or log_due_to_failure(context) diff --git a/extensions/elasticsearch/tests/features/containers/elastic_base_container.py b/extensions/elasticsearch/tests/features/containers/elastic_base_container.py index 577160c135..8859ae75ab 100644 --- a/extensions/elasticsearch/tests/features/containers/elastic_base_container.py +++ b/extensions/elasticsearch/tests/features/containers/elastic_base_container.py @@ -13,8 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -from minifi_behave.core.helpers import wait_for_condition, retry_check from minifi_behave.containers.container_linux import LinuxContainer +from minifi_behave.core.helpers import retry_check, wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext @@ -29,19 +29,44 @@ def deploy(self, context: MinifiTestContext | None, finished_str: str): condition=lambda: finished_str in self.get_logs(), timeout_seconds=300, bail_condition=lambda: self.exited, - context=context) + context=context, + ) def create_doc_elasticsearch(self, index_name: str, doc_id: str) -> bool: - (code, output) = self.exec_run(["/bin/bash", "-c", - "curl -s -u elastic:password -k -XPUT https://localhost:9200/" + index_name + "/_doc/" - + doc_id + " -H Content-Type:application/json -d'{\"field1\":\"value1\"}'"]) + (code, output) = self.exec_run( + [ + "/bin/bash", + "-c", + "curl -s -u elastic:password -k -XPUT https://localhost:9200/" + + index_name + + "/_doc/" + + doc_id + + ' -H Content-Type:application/json -d\'{"field1":"value1"}\'', + ] + ) return code == 0 and ('"_id":"' + doc_id + '"') in output def check_elastic_field_value(self, index_name: str, doc_id: str, field_name: str, field_value: str) -> bool: - (code, output) = self.exec_run(["/bin/bash", "-c", "curl -s -u elastic:password -k -XGET https://localhost:9200/" + index_name + "/_doc/" + doc_id]) + (code, output) = self.exec_run( + [ + "/bin/bash", + "-c", + "curl -s -u elastic:password -k -XGET https://localhost:9200/" + index_name + "/_doc/" + doc_id, + ] + ) return code == 0 and (field_name + '":"' + field_value) in output @retry_check() def is_elasticsearch_empty(self) -> bool: - (code, output) = self.exec_run(["curl", "-s", "-u", "elastic:password", "-k", "-XGET", "https://localhost:9200/_search"]) + (code, output) = self.exec_run( + [ + "curl", + "-s", + "-u", + "elastic:password", + "-k", + "-XGET", + "https://localhost:9200/_search", + ] + ) return code == 0 and '"hits":[]' in output diff --git a/extensions/elasticsearch/tests/features/containers/elasticsearch_container.py b/extensions/elasticsearch/tests/features/containers/elasticsearch_container.py index 45403de22d..b716f4b8fc 100644 --- a/extensions/elasticsearch/tests/features/containers/elasticsearch_container.py +++ b/extensions/elasticsearch/tests/features/containers/elasticsearch_container.py @@ -16,41 +16,89 @@ import json import logging import os +from pathlib import Path from elastic_base_container import ElasticBaseContainer -from pathlib import Path -from minifi_behave.core.ssl_utils import make_server_cert, make_cert_without_extended_usage, dump_cert, dump_key from minifi_behave.containers.file import File from minifi_behave.containers.host_file import HostFile from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import ( + dump_cert, + dump_key, + make_cert_without_extended_usage, + make_server_cert, +) + +logger = logging.getLogger(__name__) class ElasticsearchContainer(ElasticBaseContainer): IMAGE = "elasticsearch:9.1.5" def __init__(self, test_context: MinifiTestContext): - super().__init__(test_context, ElasticsearchContainer.IMAGE, f"elasticsearch-{test_context.scenario_id}") + super().__init__( + test_context, + ElasticsearchContainer.IMAGE, + f"elasticsearch-{test_context.scenario_id}", + ) http_cert, http_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) - transport_cert, transport_key = make_cert_without_extended_usage(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + transport_cert, transport_key = make_cert_without_extended_usage( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) root_ca_content = dump_cert(test_context.root_ca_cert) - self.files.append(File("/usr/share/elasticsearch/config/certs/root_ca.crt", root_ca_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/elasticsearch/config/certs/root_ca.crt", + root_ca_content, + permissions=0o644, + ) + ) http_cert_content = dump_cert(http_cert) - self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_http.crt", http_cert_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/elasticsearch/config/certs/elastic_http.crt", + http_cert_content, + permissions=0o644, + ) + ) http_key_content = dump_key(http_key) - self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_http.key", http_key_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/elasticsearch/config/certs/elastic_http.key", + http_key_content, + permissions=0o644, + ) + ) transport_cert_content = dump_cert(transport_cert) - self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_transport.crt", transport_cert_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/elasticsearch/config/certs/elastic_transport.crt", + transport_cert_content, + permissions=0o644, + ) + ) transport_key_content = dump_key(transport_key) - self.files.append(File("/usr/share/elasticsearch/config/certs/elastic_transport.key", transport_key_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/elasticsearch/config/certs/elastic_transport.key", + transport_key_content, + permissions=0o644, + ) + ) features_dir = Path(__file__).resolve().parent.parent - self.host_files.append(HostFile('/usr/share/elasticsearch/config/elasticsearch.yml', os.path.join(features_dir, "resources", "elasticsearch.yml"))) + self.host_files.append( + HostFile( + "/usr/share/elasticsearch/config/elasticsearch.yml", + os.path.join(features_dir, "resources", "elasticsearch.yml"), + ) + ) self.environment.append("ELASTIC_PASSWORD=password") @@ -62,37 +110,33 @@ def elastic_generate_apikey(self): api_user = "elastic:password" api_headers = "Content-Type:application/json" api_data = ( - '{' + "{" ' "name": "my-api-key",' ' "expiration": "1d",' ' "role_descriptors": {' ' "role-a": {' ' "cluster": [' ' "all"' - ' ],' + " ]," ' "index": [' - ' {' + " {" ' "names": [' ' "my_index"' - ' ],' + " ]," ' "privileges": [' ' "all"' - ' ]' - ' }' - ' ]' - ' }' - ' }' - '}' - ) - curl_cmd = ( - f"curl -s -u {api_user} -k -XPOST {api_url} " - f"-H {api_headers} " - f"-d'{api_data}'" + " ]" + " }" + " ]" + " }" + " }" + "}" ) + curl_cmd = f"curl -s -u {api_user} -k -XPOST {api_url} -H {api_headers} -d'{api_data}'" (code, output) = self.exec_run(["/bin/bash", "-c", curl_cmd]) if code != 0: return None - logging.info(f"Elasticsearch generate API key output: {output}") + logger.info(f"Elasticsearch generate API key output: {output}") output_lines = output.splitlines() result = json.loads(output_lines[-1]) return result["encoded"] diff --git a/extensions/elasticsearch/tests/features/containers/opensearch_container.py b/extensions/elasticsearch/tests/features/containers/opensearch_container.py index 1f09d0cce4..996fa389de 100644 --- a/extensions/elasticsearch/tests/features/containers/opensearch_container.py +++ b/extensions/elasticsearch/tests/features/containers/opensearch_container.py @@ -13,39 +13,70 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os import logging +import os +from pathlib import Path from elastic_base_container import ElasticBaseContainer -from pathlib import Path -from minifi_behave.core.ssl_utils import make_server_cert, dump_cert, dump_key from minifi_behave.containers.file import File from minifi_behave.containers.host_file import HostFile from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert + +logger = logging.getLogger(__name__) class OpensearchContainer(ElasticBaseContainer): IMAGE = "opensearchproject/opensearch:2.6.0" def __init__(self, test_context: MinifiTestContext): - super().__init__(test_context, OpensearchContainer.IMAGE, f"opensearch-{test_context.scenario_id}") + super().__init__( + test_context, + OpensearchContainer.IMAGE, + f"opensearch-{test_context.scenario_id}", + ) - admin_pem, admin_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + admin_pem, admin_key = make_server_cert( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) root_ca_content = dump_cert(test_context.root_ca_cert) - self.files.append(File("/usr/share/opensearch/config/root-ca.pem", root_ca_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/opensearch/config/root-ca.pem", + root_ca_content, + permissions=0o644, + ) + ) admin_pem_content = dump_cert(admin_pem) - self.files.append(File("/usr/share/opensearch/config/admin.pem", admin_pem_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/opensearch/config/admin.pem", + admin_pem_content, + permissions=0o644, + ) + ) admin_key_content = dump_key(admin_key) - self.files.append(File("/usr/share/opensearch/config/admin-key.pem", admin_key_content, permissions=0o644)) + self.files.append( + File( + "/usr/share/opensearch/config/admin-key.pem", + admin_key_content, + permissions=0o644, + ) + ) features_dir = Path(__file__).resolve().parent.parent - self.host_files.append(HostFile('/usr/share/opensearch/config/opensearch.yml', os.path.join(features_dir, "resources", "opensearch.yml"))) + self.host_files.append( + HostFile( + "/usr/share/opensearch/config/opensearch.yml", + os.path.join(features_dir, "resources", "opensearch.yml"), + ) + ) def deploy(self, context: MinifiTestContext | None) -> bool: - return super().deploy(context, 'Hot-reloading of audit configuration is enabled') + return super().deploy(context, "Hot-reloading of audit configuration is enabled") def add_elastic_user_to_opensearch(self): curl_cmd = [ @@ -55,9 +86,9 @@ def add_elastic_user_to_opensearch(self): "-XPUT", f"https://{self.container_name}:9200/_plugins/_security/api/internalusers/elastic", "-H 'Content-Type:application/json'", - "-d '{\"password\":\"password\",\"backend_roles\":[\"admin\"]}'" + '-d \'{"password":"password","backend_roles":["admin"]}\'', ] full_cmd = " ".join(curl_cmd) (code, output) = self.exec_run(["/bin/bash", "-c", full_cmd]) - logging.info(f"Add elastic user to Opensearch output: {output}") + logger.info(f"Add elastic user to Opensearch output: {output}") return code == 0 and '"status":"CREATED"' in output diff --git a/extensions/elasticsearch/tests/features/environment.py b/extensions/elasticsearch/tests/features/environment.py index eaea25090c..2878469ce0 100644 --- a/extensions/elasticsearch/tests/features/environment.py +++ b/extensions/elasticsearch/tests/features/environment.py @@ -13,11 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker from containers.elasticsearch_container import ElasticsearchContainer from containers.opensearch_container import OpensearchContainer -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_feature(context, feature): diff --git a/extensions/elasticsearch/tests/features/steps/steps.py b/extensions/elasticsearch/tests/features/steps/steps.py index 966c7a3b11..73dd8ea96c 100644 --- a/extensions/elasticsearch/tests/features/steps/steps.py +++ b/extensions/elasticsearch/tests/features/steps/steps.py @@ -12,47 +12,63 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from behave import step, given, then - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.minifi.controller_service import ControllerService -from minifi_behave.core.helpers import log_due_to_failure +from behave import given, step, then from containers.elasticsearch_container import ElasticsearchContainer from containers.opensearch_container import OpensearchContainer +from minifi_behave.core.helpers import log_due_to_failure +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.minifi.controller_service import ControllerService +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) -@step('an Elasticsearch server is set up and running') +@step("an Elasticsearch server is set up and running") @step('an Elasticsearch server is set up and a single document is present with "preloaded_id" in "my_index"') -@step('an Elasticsearch server is set up and a single document is present with "preloaded_id" in "my_index" with "value1" in "field1"') +@step( + 'an Elasticsearch server is set up and a single document is present with "preloaded_id" in "my_index" with "value1" in "field1"' +) def setup_elasticsearch_server(context: MinifiTestContext): context.containers["elasticsearch"] = ElasticsearchContainer(context) assert context.containers["elasticsearch"].deploy(context) - assert context.containers["elasticsearch"].create_doc_elasticsearch("my_index", "preloaded_id") or context.containers["elasticsearch"].log_app_output() + assert ( + context.containers["elasticsearch"].create_doc_elasticsearch("my_index", "preloaded_id") + or context.containers["elasticsearch"].log_app_output() + ) -@given('an ElasticsearchCredentialsControllerService is set up with Basic Authentication') +@given("an ElasticsearchCredentialsControllerService is set up with Basic Authentication") def setup_elasticsearch_credentials_service_basic_auth(context: MinifiTestContext): - controller_service = ControllerService(class_name="ElasticsearchCredentialsControllerService", service_name="ElasticsearchCredentialsControllerService") + controller_service = ControllerService( + class_name="ElasticsearchCredentialsControllerService", + service_name="ElasticsearchCredentialsControllerService", + ) controller_service.add_property("Username", "elastic") controller_service.add_property("Password", "password") context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) -@given('an ElasticsearchCredentialsControllerService is set up with ApiKey') +@given("an ElasticsearchCredentialsControllerService is set up with ApiKey") def setup_elasticsearch_credentials_service_api_key(context: MinifiTestContext): - controller_service = ControllerService(class_name="ElasticsearchCredentialsControllerService", service_name="ElasticsearchCredentialsControllerService") + controller_service = ControllerService( + class_name="ElasticsearchCredentialsControllerService", + service_name="ElasticsearchCredentialsControllerService", + ) api_key = context.containers["elasticsearch"].elastic_generate_apikey() controller_service.add_property("API Key", api_key) context.get_or_create_default_minifi_container().flow_definition.controller_services.append(controller_service) @then('Elasticsearch has a document with "{doc_id}" in "{index}" that has "{value}" set in "{field}"') -def verify_elasticsearch_document_field_value(context: MinifiTestContext, doc_id: str, index: str, value: str, field: str): - assert context.containers["elasticsearch"].check_elastic_field_value(index_name=index, doc_id=doc_id, field_name=field, field_value=value) or log_due_to_failure(context) +def verify_elasticsearch_document_field_value( + context: MinifiTestContext, doc_id: str, index: str, value: str, field: str +): + assert context.containers["elasticsearch"].check_elastic_field_value( + index_name=index, doc_id=doc_id, field_name=field, field_value=value + ) or log_due_to_failure(context) @then("Elasticsearch is empty") @@ -60,9 +76,11 @@ def verify_elasticsearch_is_empty(context): assert context.containers["elasticsearch"].is_elasticsearch_empty() or log_due_to_failure(context) -@given('an Opensearch server is set up and running') +@given("an Opensearch server is set up and running") @given('an Opensearch server is set up and a single document is present with "preloaded_id" in "my_index"') -@given('an Opensearch server is set up and a single document is present with "preloaded_id" in "my_index" with "value1" in "field1"') +@given( + 'an Opensearch server is set up and a single document is present with "preloaded_id" in "my_index" with "value1" in "field1"' +) def setup_opensearch_server(context): context.containers["opensearch"] = OpensearchContainer(context) context.containers["opensearch"].deploy(context) @@ -72,7 +90,9 @@ def setup_opensearch_server(context): @then('Opensearch has a document with "{doc_id}" in "{index}" that has "{value}" set in "{field}"') def verify_opensearch_document_field_value(context: MinifiTestContext, doc_id: str, index: str, value: str, field: str): - assert context.containers["opensearch"].check_elastic_field_value(index_name=index, doc_id=doc_id, field_name=field, field_value=value) or log_due_to_failure(context) + assert context.containers["opensearch"].check_elastic_field_value( + index_name=index, doc_id=doc_id, field_name=field, field_value=value + ) or log_due_to_failure(context) @then("Opensearch is empty") diff --git a/extensions/gcp/tests/features/containers/fake_gcs_server_container.py b/extensions/gcp/tests/features/containers/fake_gcs_server_container.py index 506478932f..28bee8d391 100644 --- a/extensions/gcp/tests/features/containers/fake_gcs_server_container.py +++ b/extensions/gcp/tests/features/containers/fake_gcs_server_container.py @@ -14,18 +14,25 @@ # limitations under the License. import logging -from minifi_behave.core.helpers import wait_for_condition, retry_check + from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.containers.directory import Directory +from minifi_behave.core.helpers import retry_check, wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +logger = logging.getLogger(__name__) + class FakeGcsServerContainer(LinuxContainer): IMAGE = "fsouza/fake-gcs-server:1.45.1" def __init__(self, test_context: MinifiTestContext): - super().__init__(FakeGcsServerContainer.IMAGE, f"fake-gcs-server-{test_context.scenario_id}", test_context.network, - command=f'-scheme http -host fake-gcs-server-{test_context.scenario_id}') + super().__init__( + FakeGcsServerContainer.IMAGE, + f"fake-gcs-server-{test_context.scenario_id}", + test_context.network, + command=f"-scheme http -host fake-gcs-server-{test_context.scenario_id}", + ) self.dirs.append(Directory(path="/data/test-bucket", files={"test-file": "preloaded data\n"})) def deploy(self, context: MinifiTestContext | None) -> bool: @@ -35,16 +42,17 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=30, bail_condition=lambda: self.exited, - context=context) + context=context, + ) @retry_check() def check_google_cloud_storage(self, content): (code, output) = self.exec_run(["grep", "-r", content, "/storage"]) - logging.info(f"GCS storage contents matching '{content}': {output}") + logger.info(f"GCS storage contents matching '{content}': {output}") return code == 0 @retry_check() def is_gcs_bucket_empty(self): (code, output) = self.exec_run(["ls", "/storage/test-bucket"]) - logging.info(f"GCS bucket contents: {output}") + logger.info(f"GCS bucket contents: {output}") return code == 0 and output == "" diff --git a/extensions/gcp/tests/features/environment.py b/extensions/gcp/tests/features/environment.py index 057b87c71e..c4e51f658f 100644 --- a/extensions/gcp/tests/features/environment.py +++ b/extensions/gcp/tests/features/environment.py @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker from containers.fake_gcs_server_container import FakeGcsServerContainer -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_all(context): diff --git a/extensions/gcp/tests/features/steps/steps.py b/extensions/gcp/tests/features/steps/steps.py index b2e9308ece..d7dbc7d78c 100644 --- a/extensions/gcp/tests/features/steps/steps.py +++ b/extensions/gcp/tests/features/steps/steps.py @@ -13,14 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. from behave import step, then - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.helpers import log_due_to_failure from containers.fake_gcs_server_container import FakeGcsServerContainer +from minifi_behave.core.helpers import log_due_to_failure +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) @step("a Google Cloud storage server is set up") @@ -31,7 +32,7 @@ def setup_gcs_server(context: MinifiTestContext): assert context.containers["fake-gcs-server"].deploy(context) -@then('an object with the content \"{content}\" is present in the Google Cloud storage') +@then('an object with the content "{content}" is present in the Google Cloud storage') def verify_gcs_object_content(context: MinifiTestContext, content: str): assert context.containers["fake-gcs-server"].check_google_cloud_storage(content) or log_due_to_failure(context) diff --git a/extensions/grafana-loki/tests/features/containers/grafana_loki_container.py b/extensions/grafana-loki/tests/features/containers/grafana_loki_container.py index d626468f77..2a5c97b742 100644 --- a/extensions/grafana-loki/tests/features/containers/grafana_loki_container.py +++ b/extensions/grafana-loki/tests/features/containers/grafana_loki_container.py @@ -15,12 +15,14 @@ import logging -from minifi_behave.core.helpers import wait_for_condition, retry_check +from docker.errors import ContainerError from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.containers.file import File +from minifi_behave.core.helpers import retry_check, wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.ssl_utils import make_server_cert, dump_cert, dump_key -from docker.errors import ContainerError +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert + +logger = logging.getLogger(__name__) class GrafanaLokiOptions: @@ -33,10 +35,16 @@ class GrafanaLokiContainer(LinuxContainer): IMAGE = "grafana/loki:3.2.1" def __init__(self, test_context: MinifiTestContext, options: GrafanaLokiOptions): - super().__init__(GrafanaLokiContainer.IMAGE, f"grafana-loki-server-{test_context.scenario_id}", test_context.network) + super().__init__( + GrafanaLokiContainer.IMAGE, + f"grafana-loki-server-{test_context.scenario_id}", + test_context.network, + ) extra_ssl_settings = "" if options.enable_ssl: - grafana_loki_cert, grafana_loki_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + grafana_loki_cert, grafana_loki_key = make_server_cert( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) root_ca_content = dump_cert(test_context.root_ca_cert) self.files.append(File("/etc/loki/root_ca.crt", root_ca_content, permissions=0o644)) @@ -89,8 +97,8 @@ def __init__(self, test_context: MinifiTestContext, options: GrafanaLokiOptions) tls_insecure_skip_verify: true """ - grafana_loki_yml_content = """ -auth_enabled: {enable_multi_tenancy} + grafana_loki_yml_content = f""" +auth_enabled: {options.enable_multi_tenancy} server: http_listen_port: 3100 @@ -123,9 +131,15 @@ def __init__(self, test_context: MinifiTestContext, options: GrafanaLokiOptions) analytics: reporting_enabled: false -""".format(extra_ssl_settings=extra_ssl_settings, enable_multi_tenancy=options.enable_multi_tenancy) +""" - self.files.append(File("/etc/loki/local-config.yaml", grafana_loki_yml_content.encode(), permissions=0o644)) + self.files.append( + File( + "/etc/loki/local-config.yaml", + grafana_loki_yml_content.encode(), + permissions=0o644, + ) + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -134,19 +148,36 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=120, bail_condition=lambda: self.exited, - context=context) + context=context, + ) @retry_check() def are_lines_present(self, lines: str, timeout: int, ssl: bool, tenant_id: str = "") -> bool: try: - self.client.containers.run("minifi-grafana-loki-helper:latest", ["python", "/scripts/check_log_lines_on_grafana.py", self.container_name, lines, str(timeout), str(ssl), tenant_id], - remove=True, stdout=True, stderr=True, network=self.network.name) + self.client.containers.run( + "minifi-grafana-loki-helper:latest", + [ + "python", + "/scripts/check_log_lines_on_grafana.py", + self.container_name, + lines, + str(timeout), + str(ssl), + tenant_id, + ], + remove=True, + stdout=True, + stderr=True, + network=self.network.name, + ) return True except ContainerError as e: stdout = e.stdout.decode("utf-8", errors="replace") if hasattr(e, "stdout") and e.stdout else "" stderr = e.stderr.decode("utf-8", errors="replace") if hasattr(e, "stderr") and e.stderr else "" - logging.error(f"Failed to run python command in grafana loki helper docker with error: '{e}', stdout: '{stdout}', stderr: '{stderr}'") + logger.error( + f"Failed to run python command in grafana loki helper docker with error: '{e}', stdout: '{stdout}', stderr: '{stderr}'" + ) return False except Exception as e: - logging.error(f"Unexpected error while running python command in grafana loki helper docker: '{e}'") + logger.error(f"Unexpected error while running python command in grafana loki helper docker: '{e}'") return False diff --git a/extensions/grafana-loki/tests/features/containers/reverse_proxy_container.py b/extensions/grafana-loki/tests/features/containers/reverse_proxy_container.py index f622cbda3e..a24a131841 100644 --- a/extensions/grafana-loki/tests/features/containers/reverse_proxy_container.py +++ b/extensions/grafana-loki/tests/features/containers/reverse_proxy_container.py @@ -13,14 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -from minifi_behave.core.minifi_test_context import MinifiTestContext from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.core.helpers import wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext class ReverseProxyContainer(LinuxContainer): def __init__(self, test_context: MinifiTestContext): - super().__init__("minifi-reverse-proxy:latest", f"reverse-proxy-{test_context.scenario_id}", test_context.network) + super().__init__( + "minifi-reverse-proxy:latest", + f"reverse-proxy-{test_context.scenario_id}", + test_context.network, + ) self.environment = [ "BASIC_USERNAME=admin", "BASIC_PASSWORD=password", @@ -35,4 +39,5 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=60, bail_condition=lambda: self.exited, - context=context) + context=context, + ) diff --git a/extensions/grafana-loki/tests/features/environment.py b/extensions/grafana-loki/tests/features/environment.py index 1318d2f5de..34491c22b8 100644 --- a/extensions/grafana-loki/tests/features/environment.py +++ b/extensions/grafana-loki/tests/features/environment.py @@ -13,12 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker -from containers.grafana_loki_container import GrafanaLokiContainer from pathlib import Path + +from containers.grafana_loki_container import GrafanaLokiContainer from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_all(context): @@ -33,13 +34,13 @@ def before_all(context): grafana_helper_builder = DockerImageBuilder( image_tag="minifi-grafana-loki-helper:latest", dockerfile_content=dockerfile, - files_on_context={"check_log_lines_on_grafana.py": check_log_lines_content} + files_on_context={"check_log_lines_on_grafana.py": check_log_lines_content}, ) grafana_helper_builder.build() reverse_proxy_builder = DockerImageBuilder( image_tag="minifi-reverse-proxy:latest", - build_context_path=str(Path(__file__).resolve().parent / "resources" / "reverse-proxy") + build_context_path=str(Path(__file__).resolve().parent / "resources" / "reverse-proxy"), ) reverse_proxy_builder.build() docker.from_env().images.pull(GrafanaLokiContainer.IMAGE) diff --git a/extensions/grafana-loki/tests/features/resources/check_log_lines_on_grafana.py b/extensions/grafana-loki/tests/features/resources/check_log_lines_on_grafana.py index 6d961a256f..fcae15ef38 100644 --- a/extensions/grafana-loki/tests/features/resources/check_log_lines_on_grafana.py +++ b/extensions/grafana-loki/tests/features/resources/check_log_lines_on_grafana.py @@ -14,9 +14,10 @@ # limitations under the License. import sys -import requests import time +import requests + def wait_for(action, timeout_seconds, *args, **kwargs) -> bool: start_time = time.perf_counter() @@ -40,7 +41,7 @@ def verify_log_lines_on_grafana_loki(host: str, lines: list[str], ssl: bool, ten headers = None if tenant_id: - headers = {'X-Scope-OrgID': tenant_id} + headers = {"X-Scope-OrgID": tenant_id} response = requests.get(query_url, verify=False, timeout=30, headers=headers) if response.status_code < 200 or response.status_code >= 300: @@ -60,8 +61,13 @@ def verify_log_lines_on_grafana_loki(host: str, lines: list[str], ssl: bool, ten return True -def wait_for_lines_on_grafana_loki(host: str, lines: list[str], timeout_seconds: int, ssl: bool, tenant_id: str) -> bool: - return wait_for(lambda: verify_log_lines_on_grafana_loki(host, lines, ssl, tenant_id), timeout_seconds) +def wait_for_lines_on_grafana_loki( + host: str, lines: list[str], timeout_seconds: int, ssl: bool, tenant_id: str +) -> bool: + return wait_for( + lambda: verify_log_lines_on_grafana_loki(host, lines, ssl, tenant_id), + timeout_seconds, + ) if __name__ == "__main__": diff --git a/extensions/grafana-loki/tests/features/steps/steps.py b/extensions/grafana-loki/tests/features/steps/steps.py index 0ab0769238..45c1ecb9fd 100644 --- a/extensions/grafana-loki/tests/features/steps/steps.py +++ b/extensions/grafana-loki/tests/features/steps/steps.py @@ -13,15 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. from behave import step, then - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.helpers import log_due_to_failure from containers.grafana_loki_container import GrafanaLokiContainer, GrafanaLokiOptions from containers.reverse_proxy_container import ReverseProxyContainer +from minifi_behave.core.helpers import log_due_to_failure +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) @step("a Grafana Loki server is set up") @@ -31,7 +32,9 @@ def setup_grafana_loki_server(context: MinifiTestContext): @step("a Grafana Loki server is set up with multi-tenancy enabled") def setup_grafana_loki_server_multitenant(context: MinifiTestContext): - context.containers["grafana-loki-server"] = GrafanaLokiContainer(context, GrafanaLokiOptions(enable_multi_tenancy=True)) + context.containers["grafana-loki-server"] = GrafanaLokiContainer( + context, GrafanaLokiOptions(enable_multi_tenancy=True) + ) @step("a Grafana Loki server with SSL is set up") @@ -39,25 +42,35 @@ def setup_grafana_loki_server_ssl(context: MinifiTestContext): context.containers["grafana-loki-server"] = GrafanaLokiContainer(context, GrafanaLokiOptions(enable_ssl=True)) -@then("\"{lines}\" lines are published to the Grafana Loki server in less than {timeout_seconds:d} seconds") -@then("\"{lines}\" line is published to the Grafana Loki server in less than {timeout_seconds:d} seconds") +@then('"{lines}" lines are published to the Grafana Loki server in less than {timeout_seconds:d} seconds') +@then('"{lines}" line is published to the Grafana Loki server in less than {timeout_seconds:d} seconds') def verify_lines_published_to_loki(context, lines: str, timeout_seconds: int): - assert context.containers["grafana-loki-server"].are_lines_present(lines, timeout_seconds, ssl=False) or log_due_to_failure(context) + assert context.containers["grafana-loki-server"].are_lines_present( + lines, timeout_seconds, ssl=False + ) or log_due_to_failure(context) -@then("\"{lines}\" lines are published to the \"{tenant_id}\" tenant on the Grafana Loki server in less than {timeout_seconds:d} seconds") -@then("\"{lines}\" line is published to the \"{tenant_id}\" tenant on the Grafana Loki server in less than {timeout_seconds:d} seconds") +@then( + '"{lines}" lines are published to the "{tenant_id}" tenant on the Grafana Loki server in less than {timeout_seconds:d} seconds' +) +@then( + '"{lines}" line is published to the "{tenant_id}" tenant on the Grafana Loki server in less than {timeout_seconds:d} seconds' +) def verify_lines_published_to_loki_tenant(context, lines: str, timeout_seconds: int, tenant_id: str): - assert context.containers["grafana-loki-server"].are_lines_present(lines, timeout_seconds, ssl=False, tenant_id=tenant_id) or log_due_to_failure(context) + assert context.containers["grafana-loki-server"].are_lines_present( + lines, timeout_seconds, ssl=False, tenant_id=tenant_id + ) or log_due_to_failure(context) -@then("\"{lines}\" lines are published using SSL to the Grafana Loki server in less than {timeout_seconds:d} seconds") -@then("\"{lines}\" line is published using SSL to the Grafana Loki server in less than {timeout_seconds:d} seconds") +@then('"{lines}" lines are published using SSL to the Grafana Loki server in less than {timeout_seconds:d} seconds') +@then('"{lines}" line is published using SSL to the Grafana Loki server in less than {timeout_seconds:d} seconds') def verify_lines_published_to_loki_ssl(context, lines: str, timeout_seconds: int): - assert context.containers["grafana-loki-server"].are_lines_present(lines, timeout_seconds, ssl=True) or log_due_to_failure(context) + assert context.containers["grafana-loki-server"].are_lines_present( + lines, timeout_seconds, ssl=True + ) or log_due_to_failure(context) # Nginx reverse proxy -@step('a reverse proxy is set up to forward requests to the Grafana Loki server') +@step("a reverse proxy is set up to forward requests to the Grafana Loki server") def setup_reverse_proxy_for_loki(context): context.containers["reverse-proxy"] = ReverseProxyContainer(context) diff --git a/extensions/kafka/tests/features/containers/kafka_server_container.py b/extensions/kafka/tests/features/containers/kafka_server_container.py index d4434cd042..5181db6590 100644 --- a/extensions/kafka/tests/features/containers/kafka_server_container.py +++ b/extensions/kafka/tests/features/containers/kafka_server_container.py @@ -16,21 +16,26 @@ import logging import re -from cryptography.hazmat.primitives import serialization import jks - -from minifi_behave.core.helpers import wait_for_condition -from minifi_behave.core.ssl_utils import make_server_cert, dump_cert, dump_key +from cryptography.hazmat.primitives import serialization from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.containers.file import File +from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert + +logger = logging.getLogger(__name__) class KafkaServer(LinuxContainer): IMAGE = "apache/kafka:4.1.0" def __init__(self, test_context: MinifiTestContext): - super().__init__(KafkaServer.IMAGE, f"kafka-server-{test_context.scenario_id}", test_context.network) + super().__init__( + KafkaServer.IMAGE, + f"kafka-server-{test_context.scenario_id}", + test_context.network, + ) self.environment.append("KAFKA_NODE_ID=1") self.environment.append("KAFKA_PROCESS_ROLES=controller,broker") self.environment.append("KAFKA_INTER_BROKER_LISTENER_NAME=PLAINTEXT") @@ -39,12 +44,20 @@ def __init__(self, test_context: MinifiTestContext): self.environment.append("KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1") self.environment.append("KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1") self.environment.append(f"KAFKA_CONTROLLER_QUORUM_VOTERS=1@kafka-server-{test_context.scenario_id}:9096") - self.environment.append(f"KAFKA_LISTENERS=PLAINTEXT://kafka-server-{test_context.scenario_id}:9092,SASL_PLAINTEXT://kafka-server-{test_context.scenario_id}:9094,SSL://kafka-server-{test_context.scenario_id}:9093,SASL_SSL://kafka-server-{test_context.scenario_id}:9095,CONTROLLER://kafka-server-{test_context.scenario_id}:9096") - self.environment.append(f"KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-server-{test_context.scenario_id}:9092,SASL_PLAINTEXT://kafka-server-{test_context.scenario_id}:9094,SSL://kafka-server-{test_context.scenario_id}:9093,SASL_SSL://kafka-server-{test_context.scenario_id}:9095,CONTROLLER://kafka-server-{test_context.scenario_id}:9096") - self.environment.append("KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT,SSL:SSL,SASL_SSL:SASL_SSL,CONTROLLER:PLAINTEXT") + self.environment.append( + f"KAFKA_LISTENERS=PLAINTEXT://kafka-server-{test_context.scenario_id}:9092,SASL_PLAINTEXT://kafka-server-{test_context.scenario_id}:9094,SSL://kafka-server-{test_context.scenario_id}:9093,SASL_SSL://kafka-server-{test_context.scenario_id}:9095,CONTROLLER://kafka-server-{test_context.scenario_id}:9096" + ) + self.environment.append( + f"KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://kafka-server-{test_context.scenario_id}:9092,SASL_PLAINTEXT://kafka-server-{test_context.scenario_id}:9094,SSL://kafka-server-{test_context.scenario_id}:9093,SASL_SSL://kafka-server-{test_context.scenario_id}:9095,CONTROLLER://kafka-server-{test_context.scenario_id}:9096" + ) + self.environment.append( + "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,SASL_PLAINTEXT:SASL_PLAINTEXT,SSL:SSL,SASL_SSL:SASL_SSL,CONTROLLER:PLAINTEXT" + ) self.environment.append("KAFKA_SASL_MECHANISM_INTER_BROKER_PROTOCOL=PLAIN") self.environment.append("KAFKA_SASL_ENABLED_MECHANISMS=PLAIN") - self.environment.append("KAFKA_OPTS=-Djava.security.auth.login.config=/opt/kafka/config/kafka_jaas.conf -Dlog4j2.rootLogger.level=DEBUG -Dlog4j2.logger.org.apache.kafka.controller.level=DEBUG") + self.environment.append( + "KAFKA_OPTS=-Djava.security.auth.login.config=/opt/kafka/config/kafka_jaas.conf -Dlog4j2.rootLogger.level=DEBUG -Dlog4j2.logger.org.apache.kafka.controller.level=DEBUG" + ) self.environment.append("KAFKA_SSL_PROTOCOL=TLS") self.environment.append("KAFKA_SSL_ENABLED_PROTOCOLS=TLSv1.2") self.environment.append("KAFKA_SSL_KEYSTORE_TYPE=JKS") @@ -56,23 +69,42 @@ def __init__(self, test_context: MinifiTestContext): self.environment.append("KAFKA_SSL_TRUSTSTORE_FILENAME=kafka.truststore.jks") self.environment.append("KAFKA_SSL_CLIENT_AUTH=none") - kafka_cert, kafka_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + kafka_cert, kafka_key = make_server_cert( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) - pke = jks.PrivateKeyEntry.new('kafka-broker-cert', [dump_cert(kafka_cert, encoding_type=serialization.Encoding.DER)], dump_key(kafka_key, encoding_type=serialization.Encoding.DER), 'rsa_raw') - server_keystore = jks.KeyStore.new('jks', [pke]) - server_keystore_content = server_keystore.saves('abcdefgh') - self.files.append(File("/etc/kafka/secrets/kafka.keystore.jks", server_keystore_content, permissions=0o644)) - self.files.append(File("/etc/kafka/secrets/credentials.conf", b'abcdefgh', permissions=0o644)) + pke = jks.PrivateKeyEntry.new( + "kafka-broker-cert", + [dump_cert(kafka_cert, encoding_type=serialization.Encoding.DER)], + dump_key(kafka_key, encoding_type=serialization.Encoding.DER), + "rsa_raw", + ) + server_keystore = jks.KeyStore.new("jks", [pke]) + server_keystore_content = server_keystore.saves("abcdefgh") + self.files.append( + File( + "/etc/kafka/secrets/kafka.keystore.jks", + server_keystore_content, + permissions=0o644, + ) + ) + self.files.append(File("/etc/kafka/secrets/credentials.conf", b"abcdefgh", permissions=0o644)) trusted_cert = jks.TrustedCertEntry.new( - 'root-ca', # Alias for the certificate - dump_cert(test_context.root_ca_cert, encoding_type=serialization.Encoding.DER) + "root-ca", # Alias for the certificate + dump_cert(test_context.root_ca_cert, encoding_type=serialization.Encoding.DER), ) # Create a JKS keystore that will serve as a truststore with the trusted cert entry. - truststore = jks.KeyStore.new('jks', [trusted_cert]) - truststore_content = truststore.saves('abcdefgh') - self.files.append(File("/etc/kafka/secrets/kafka.truststore.jks", truststore_content, permissions=0o644)) + truststore = jks.KeyStore.new("jks", [trusted_cert]) + truststore_content = truststore.saves("abcdefgh") + self.files.append( + File( + "/etc/kafka/secrets/kafka.truststore.jks", + truststore_content, + permissions=0o644, + ) + ) jaas_config_file_content = """ KafkaServer { @@ -89,7 +121,13 @@ def __init__(self, test_context: MinifiTestContext): password="admin-secret"; }; """ - self.files.append(File("/opt/kafka/config/kafka_jaas.conf", jaas_config_file_content, permissions=0o644)) + self.files.append( + File( + "/opt/kafka/config/kafka_jaas.conf", + jaas_config_file_content, + permissions=0o644, + ) + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -98,26 +136,52 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=60, bail_condition=lambda: self.exited, - context=context) + context=context, + ) def create_topic(self, topic_name: str): - (code, output) = self.exec_run(["/bin/sh", "-c", f"/opt/kafka/bin/kafka-topics.sh --create --topic '{topic_name}' --bootstrap-server '{self.container_name}':9092"]) - logging.info(f"Create topic output: '{output}' with code {code}") + (code, output) = self.exec_run( + [ + "/bin/sh", + "-c", + f"/opt/kafka/bin/kafka-topics.sh --create --topic '{topic_name}' --bootstrap-server '{self.container_name}':9092", + ] + ) + logger.info(f"Create topic output: '{output}' with code {code}") return code == 0 def produce_message(self, topic_name: str, message: str): - (code, output) = self.exec_run(["/bin/sh", "-c", f"echo '{message}' | /opt/kafka/bin/kafka-console-producer.sh --topic '{topic_name}' --bootstrap-server '{self.container_name}':9092"]) - logging.info(f"Produce message output: '{output}' with code {code}") + (code, output) = self.exec_run( + [ + "/bin/sh", + "-c", + f"echo '{message}' | /opt/kafka/bin/kafka-console-producer.sh --topic '{topic_name}' --bootstrap-server '{self.container_name}':9092", + ] + ) + logger.info(f"Produce message output: '{output}' with code {code}") return code == 0 def produce_message_with_key(self, topic_name: str, message: str, message_key: str): - (code, output) = self.exec_run(["/bin/sh", "-c", f" echo '{message_key}:{message}' | /opt/kafka/bin/kafka-console-producer.sh --property 'key.separator=:' --property 'parse.key=true' --topic '{topic_name}' --bootstrap-server '{self.container_name}':9092"]) - logging.info(f"Produce message with key output: '{output}' with code {code}") + (code, output) = self.exec_run( + [ + "/bin/sh", + "-c", + f" echo '{message_key}:{message}' | /opt/kafka/bin/kafka-console-producer.sh --property 'key.separator=:' --property 'parse.key=true' --topic '{topic_name}' --bootstrap-server '{self.container_name}':9092", + ] + ) + logger.info(f"Produce message with key output: '{output}' with code {code}") return code == 0 def run_python_in_kafka_helper_docker(self, command: str): - output = self.client.containers.run("minifi-kafka-helper:latest", ["python", "-c", command], remove=True, stdout=True, stderr=True, network=self.network.name) - logging.info(f"Run python in kafka helper docker output: '{output.decode('utf-8')}'") + output = self.client.containers.run( + "minifi-kafka-helper:latest", + ["python", "-c", command], + remove=True, + stdout=True, + stderr=True, + network=self.network.name, + ) + logger.info(f"Run python in kafka helper docker output: '{output.decode('utf-8')}'") return True def wait_for_kafka_consumer_to_be_registered(self, expected_consumer_count: int): @@ -126,4 +190,5 @@ def wait_for_kafka_consumer_to_be_registered(self, expected_consumer_count: int) condition=lambda: len(re.findall(message_regex, self.get_logs())) >= expected_consumer_count, timeout_seconds=60, bail_condition=lambda: self.exited, - context=None) + context=None, + ) diff --git a/extensions/kafka/tests/features/environment.py b/extensions/kafka/tests/features/environment.py index d7ee24d67c..0d22bd6d23 100644 --- a/extensions/kafka/tests/features/environment.py +++ b/extensions/kafka/tests/features/environment.py @@ -13,21 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker from containers.kafka_server_container import KafkaServer from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_all(context): dockerfile = """ FROM python:3.13-slim-bookworm RUN pip install confluent-kafka""" - builder = DockerImageBuilder( - image_tag="minifi-kafka-helper:latest", - dockerfile_content=dockerfile - ) + builder = DockerImageBuilder(image_tag="minifi-kafka-helper:latest", dockerfile_content=dockerfile) builder.build() docker.from_env().images.pull(KafkaServer.IMAGE) diff --git a/extensions/kafka/tests/features/steps/steps.py b/extensions/kafka/tests/features/steps/steps.py index 7417e709f5..cf150785d5 100644 --- a/extensions/kafka/tests/features/steps/steps.py +++ b/extensions/kafka/tests/features/steps/steps.py @@ -16,15 +16,16 @@ import binascii import time -from behave import step, when, given - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from behave import given, step, when +from containers.kafka_server_container import KafkaServer from minifi_behave.core.minifi_test_context import MinifiTestContext from minifi_behave.minifi.processor import Processor -from containers.kafka_server_container import KafkaServer +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) @step("a Kafka server is set up") @@ -80,17 +81,25 @@ def initialize_kafka_topic(context: MinifiTestContext, topic_name: str): def publish_message_to_topic(context: MinifiTestContext, message: str, topic_name: str): kafka_server_container = context.containers["kafka-server"] assert isinstance(kafka_server_container, KafkaServer) - assert kafka_server_container.produce_message(topic_name=topic_name, message=message) or kafka_server_container.log_app_output() + assert ( + kafka_server_container.produce_message(topic_name=topic_name, message=message) + or kafka_server_container.log_app_output() + ) @step('a message with content "{message}" is published to the "{topic_name}" topic with key "{key}"') def publish_message_with_key_to_topic(context: MinifiTestContext, message: str, topic_name: str, key: str): kafka_server_container = context.containers["kafka-server"] assert isinstance(kafka_server_container, KafkaServer) - assert kafka_server_container.produce_message_with_key(topic_name=topic_name, message=message, message_key=key) or kafka_server_container.log_app_output() + assert ( + kafka_server_container.produce_message_with_key(topic_name=topic_name, message=message, message_key=key) + or kafka_server_container.log_app_output() + ) -@given("the \"{property_name}\" property of the {processor_name} processor is set to match {key_attribute_encoding} encoded kafka message key \"{message_key}\"") +@given( + 'the "{property_name}" property of the {processor_name} processor is set to match {key_attribute_encoding} encoded kafka message key "{message_key}"' +) def set_property_to_match_message_key(context, property_name, processor_name, key_attribute_encoding, message_key): if key_attribute_encoding.lower() == "hex": encoded_key = binascii.hexlify(message_key.encode("utf-8")).upper() @@ -103,7 +112,9 @@ def set_property_to_match_message_key(context, property_name, processor_name, ke processor.add_property(property_name, filtering) -@when("the publisher performs a {transaction_type} transaction publishing to the \"{topic_name}\" topic these messages: {messages}") +@when( + 'the publisher performs a {transaction_type} transaction publishing to the "{topic_name}" topic these messages: {messages}' +) def publish_to_topic_transaction_style(context, transaction_type, topic_name, messages): if transaction_type == "SINGLE_COMMITTED_TRANSACTION": python_code = f""" @@ -153,11 +164,16 @@ def publish_to_topic_transaction_style(context, transaction_type, topic_name, me producer.abort_transaction() """ else: - raise Exception("Unknown transaction type.") - assert context.containers["kafka-server"].run_python_in_kafka_helper_docker(python_code) or context.containers["kafka-server"].log_app_output() + raise RuntimeError("Unknown transaction type.") + assert ( + context.containers["kafka-server"].run_python_in_kafka_helper_docker(python_code) + or context.containers["kafka-server"].log_app_output() + ) -@when("a message with content \"{content}\" is published to the \"{topic_name}\" topic with headers \"{semicolon_separated_headers}\"") +@when( + 'a message with content "{content}" is published to the "{topic_name}" topic with headers "{semicolon_separated_headers}"' +) def publish_with_headers_to_topic(context, content, topic_name, semicolon_separated_headers): python_code = f""" from confluent_kafka import Producer @@ -169,10 +185,13 @@ def publish_with_headers_to_topic(context, content, topic_name, semicolon_separa producer.produce("{topic_name}", "{content}".encode("utf-8"), headers=headers) producer.flush(10) """ - assert context.containers["kafka-server"].run_python_in_kafka_helper_docker(python_code) or context.containers["kafka-server"].log_app_output() + assert ( + context.containers["kafka-server"].run_python_in_kafka_helper_docker(python_code) + or context.containers["kafka-server"].log_app_output() + ) -@when("two messages with content \"{content_one}\" and \"{content_two}\" is published to the \"{topic_name}\" topic") +@when('two messages with content "{content_one}" and "{content_two}" is published to the "{topic_name}" topic') def publish_two_messages_to_topic(context, content_one, content_two, topic_name): python_code = f""" from confluent_kafka import Producer @@ -182,10 +201,13 @@ def publish_two_messages_to_topic(context, content_one, content_two, topic_name) producer.produce("{topic_name}", "{content_two}") producer.flush(10) """ - assert context.containers["kafka-server"].run_python_in_kafka_helper_docker(python_code) or context.containers["kafka-server"].log_app_output() + assert ( + context.containers["kafka-server"].run_python_in_kafka_helper_docker(python_code) + or context.containers["kafka-server"].log_app_output() + ) -@when("{number_of_messages} kafka messages are sent to the topic \"{topic_name}\"") +@when('{number_of_messages} kafka messages are sent to the topic "{topic_name}"') def publish_batch_to_topic(context, number_of_messages, topic_name): python_code = f""" from confluent_kafka import Producer @@ -195,12 +217,18 @@ def publish_batch_to_topic(context, number_of_messages, topic_name): producer.produce("{topic_name}", str(uuid.uuid4()).encode("utf-8")) producer.flush(10) """ - assert context.containers["kafka-server"].run_python_in_kafka_helper_docker(python_code) or context.containers["kafka-server"].log_app_output() + assert ( + context.containers["kafka-server"].run_python_in_kafka_helper_docker(python_code) + or context.containers["kafka-server"].log_app_output() + ) @when("the Kafka consumer is registered in kafka broker") def wait_for_consumer_registration(context): - assert context.containers["kafka-server"].wait_for_kafka_consumer_to_be_registered(1) or context.containers["kafka-server"].log_app_output() + assert ( + context.containers["kafka-server"].wait_for_kafka_consumer_to_be_registered(1) + or context.containers["kafka-server"].log_app_output() + ) # After the consumer is registered there is still some time needed for consumer-broker synchronization # Unfortunately there are no additional log messages that could be checked for this time.sleep(2) @@ -208,7 +236,10 @@ def wait_for_consumer_registration(context): @when("the Kafka consumer is reregistered in kafka broker") def wait_for_consumer_reregistration(context): - assert context.containers["kafka-server"].wait_for_kafka_consumer_to_be_registered(2) or context.containers["kafka-server"].log_app_output() + assert ( + context.containers["kafka-server"].wait_for_kafka_consumer_to_be_registered(2) + or context.containers["kafka-server"].log_app_output() + ) # After the consumer is registered there is still some time needed for consumer-broker synchronization # Unfortunately there are no additional log messages that could be checked for this time.sleep(2) diff --git a/extensions/kubernetes/tests/features/environment.py b/extensions/kubernetes/tests/features/environment.py index 83408016a8..2420fbb056 100644 --- a/extensions/kubernetes/tests/features/environment.py +++ b/extensions/kubernetes/tests/features/environment.py @@ -13,15 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker import os -from minifi_behave.core.hooks import common_after_scenario, common_before_scenario, get_minifi_container_image +from minifi_behave.core.hooks import ( + common_after_scenario, + common_before_scenario, + get_minifi_container_image, +) from steps.kubernetes_proxy import KubernetesProxy +import docker + def before_feature(context, feature): - if "rpm" in os.environ['MINIFI_TAG_PREFIX']: + if "rpm" in os.environ["MINIFI_TAG_PREFIX"]: feature.skip("This feature is not yet supported on RPM installed images") minifi_image = docker.from_env().images.get(get_minifi_container_image()) diff --git a/extensions/kubernetes/tests/features/steps/kubernetes_proxy.py b/extensions/kubernetes/tests/features/steps/kubernetes_proxy.py index c23a164e35..62af27a1b3 100644 --- a/extensions/kubernetes/tests/features/steps/kubernetes_proxy.py +++ b/extensions/kubernetes/tests/features/steps/kubernetes_proxy.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker import glob import os import platform @@ -25,6 +24,7 @@ from pathlib import Path from textwrap import dedent +import docker KUBERNETES_CONTAINER_NAME = "kind-control-plane" @@ -34,11 +34,11 @@ def __init__(self): self.temp_directory = tempfile.TemporaryDirectory() self.resources_directory = Path(__file__).resolve().parent.parent / "resources" - self.minifi_conf_directory = os.path.join(self.temp_directory.name, 'minifi_conf') + self.minifi_conf_directory = os.path.join(self.temp_directory.name, "minifi_conf") os.mkdir(self.minifi_conf_directory) - self.kind_binary_path = os.path.join(self.temp_directory.name, 'kind') - self.kind_config_path = os.path.join(self.temp_directory.name, 'kind-config.yml') + self.kind_binary_path = os.path.join(self.temp_directory.name, "kind") + self.kind_config_path = os.path.join(self.temp_directory.name, "kind-config.yml") self.__download_kind() self.__create_kind_config() @@ -47,16 +47,22 @@ def __init__(self): def __download_kind(self): is_x86 = platform.machine() in ("i386", "AMD64", "x86_64") - download_link = 'https://kind.sigs.k8s.io/dl/v0.31.0/kind-linux-amd64' + download_link = "https://kind.sigs.k8s.io/dl/v0.31.0/kind-linux-amd64" if not is_x86: - if 'Linux' in platform.system(): - download_link = 'https://kind.sigs.k8s.io/dl/v0.31.0/kind-linux-arm64' + if "Linux" in platform.system(): + download_link = "https://kind.sigs.k8s.io/dl/v0.31.0/kind-linux-arm64" else: - download_link = 'https://kind.sigs.k8s.io/dl/v0.31.0/kind-darwin-arm64' + download_link = "https://kind.sigs.k8s.io/dl/v0.31.0/kind-darwin-arm64" if not os.path.exists(self.kind_binary_path): - if subprocess.run(['curl', '-Lo', self.kind_binary_path, download_link]).returncode != 0: - raise Exception("Could not download kind") + if ( + subprocess.run( + ["curl", "-Lo", self.kind_binary_path, download_link], + check=False, + ).returncode + != 0 + ): + raise RuntimeError("Could not download kind") os.chmod(self.kind_binary_path, stat.S_IXUSR) def __create_kind_config(self): @@ -73,8 +79,8 @@ def __create_kind_config(self): containerPath: /tmp/minifi_conf """) - with open(self.kind_config_path, 'wb') as config_file: - config_file.write(kind_config.encode('utf-8')) + with open(self.kind_config_path, "wb") as config_file: + config_file.write(kind_config.encode("utf-8")) def write_minifi_conf_file(self, file_name: str, content: str): file_path = os.path.join(self.minifi_conf_directory, file_name) @@ -82,67 +88,93 @@ def write_minifi_conf_file(self, file_name: str, content: str): file.write(content) def create_cluster(self): - if subprocess.run([self.kind_binary_path, 'create', 'cluster', '--config=' + self.kind_config_path]).returncode != 0: - raise Exception("Could not create the kind Kubernetes cluster") + if ( + subprocess.run( + [ + self.kind_binary_path, + "create", + "cluster", + "--config=" + self.kind_config_path, + ], + check=False, + ).returncode + != 0 + ): + raise RuntimeError("Could not create the kind Kubernetes cluster") self.status = "running" def delete_cluster(self): - if subprocess.run([self.kind_binary_path, 'delete', 'cluster']).returncode != 0: - raise Exception("Could not delete the kind Kubernetes cluster") + if subprocess.run([self.kind_binary_path, "delete", "cluster"], check=False).returncode != 0: + raise RuntimeError("Could not delete the kind Kubernetes cluster") self.status = "exited" def load_docker_image(self, image_name: str): - if subprocess.run([self.kind_binary_path, 'load', 'docker-image', image_name]).returncode != 0: - raise Exception("Could not load the %s docker image into the kind Kubernetes cluster" % image_name) + if ( + subprocess.run( + [self.kind_binary_path, "load", "docker-image", image_name], + check=False, + ).returncode + != 0 + ): + raise RuntimeError(f"Could not load the {image_name} docker image into the kind Kubernetes cluster") def create_helper_objects(self): - self.__wait_for_default_service_account('default') - namespaces = self.__create_objects_of_type('namespace') + self.__wait_for_default_service_account("default") + namespaces = self.__create_objects_of_type("namespace") for namespace in namespaces: self.__wait_for_default_service_account(namespace) - self.__create_objects_of_type('dependencies') - self.__create_objects_of_type('helper-pod') - self.__create_objects_of_type('clusterrole') - self.__create_objects_of_type('clusterrolebinding') + self.__create_objects_of_type("dependencies") + self.__create_objects_of_type("helper-pod") + self.__create_objects_of_type("clusterrole") + self.__create_objects_of_type("clusterrolebinding") - self.__wait_for_pod_startup('default', 'hello-world-one') - self.__wait_for_pod_startup('default', 'hello-world-two') - self.__wait_for_pod_startup('kube-system', 'metrics-server') + self.__wait_for_pod_startup("default", "hello-world-one") + self.__wait_for_pod_startup("default", "hello-world-two") + self.__wait_for_pod_startup("kube-system", "metrics-server") def create_minifi_pod(self): - self.__create_objects_of_type('test-pod') - self.__wait_for_pod_startup('daemon', 'minifi') + self.__create_objects_of_type("test-pod") + self.__wait_for_pod_startup("daemon", "minifi") def __wait_for_pod_startup(self, namespace: str, pod_name: str): for i in range(120): if i > 0: time.sleep(1) - (code, output) = self.docker_client.containers.get(KUBERNETES_CONTAINER_NAME).exec_run(['kubectl', '-n', namespace, 'get', 'pods']) - if code == 0 and re.search(f'{pod_name}.*Running', output.decode('utf-8')): + (code, output) = self.docker_client.containers.get(KUBERNETES_CONTAINER_NAME).exec_run( + ["kubectl", "-n", namespace, "get", "pods"] + ) + if code == 0 and re.search(f"{pod_name}.*Running", output.decode("utf-8")): return - raise Exception(f"The pod {namespace}:{pod_name} in the Kubernetes cluster failed to start up") + raise RuntimeError(f"The pod {namespace}:{pod_name} in the Kubernetes cluster failed to start up") def __wait_for_default_service_account(self, namespace: str): for i in range(120): if i > 0: time.sleep(1) - (code, output) = self.docker_client.containers.get(KUBERNETES_CONTAINER_NAME).exec_run(['kubectl', '-n', namespace, 'get', 'serviceaccount', 'default']) + (code, _output) = self.docker_client.containers.get(KUBERNETES_CONTAINER_NAME).exec_run( + ["kubectl", "-n", namespace, "get", "serviceaccount", "default"] + ) if code == 0: return - raise Exception("Default service account for namespace '%s' not found" % namespace) + raise RuntimeError(f"Default service account for namespace '{namespace}' not found") def __create_objects_of_type(self, type: str) -> list[str]: found_objects = [] - for full_file_name in glob.iglob(os.path.join(self.resources_directory, f'*.{type}.yml')): + for full_file_name in glob.iglob(os.path.join(self.resources_directory, f"*.{type}.yml")): file_name = os.path.basename(full_file_name) - file_name_in_container = os.path.join('/var/tmp', file_name) + file_name_in_container = os.path.join("/var/tmp", file_name) - (code, output) = self.docker_client.containers.get(KUBERNETES_CONTAINER_NAME).exec_run(['kubectl', 'apply', '-f', file_name_in_container]) + (code, output) = self.docker_client.containers.get(KUBERNETES_CONTAINER_NAME).exec_run( + ["kubectl", "apply", "-f", file_name_in_container] + ) if code != 0: - raise Exception("Could not create kubernetes object from file '%s': %s" % full_file_name, output.decode('utf-8')) + raise RuntimeError( + "Could not create kubernetes object from file '{}': {}".format(*full_file_name), + output.decode("utf-8"), + ) - object_name = file_name.replace(f'.{type}.yml', '') + object_name = file_name.replace(f".{type}.yml", "") found_objects.append(object_name) return found_objects @@ -157,8 +189,14 @@ def exec_run(self, command: str) -> tuple[int | None, bytes]: return None, b"The kind Kubernetes cluster is not running." def logs(self) -> bytes: - (code, output) = self.docker_client.containers.get(KUBERNETES_CONTAINER_NAME).exec_run(['kubectl', '-n', 'daemon', 'logs', 'minifi']) + (code, output) = self.docker_client.containers.get(KUBERNETES_CONTAINER_NAME).exec_run( + ["kubectl", "-n", "daemon", "logs", "minifi"] + ) if code == 0: return output else: - raise Exception("Could not get logs from the kind Kubernetes cluster, error %d: %s", code, output.decode('utf-8')) + raise RuntimeError( + "Could not get logs from the kind Kubernetes cluster, error %d: %s", + code, + output.decode("utf-8"), + ) diff --git a/extensions/kubernetes/tests/features/steps/minifi_as_pod_in_kubernetes_cluster.py b/extensions/kubernetes/tests/features/steps/minifi_as_pod_in_kubernetes_cluster.py index 320e896867..70d688f254 100644 --- a/extensions/kubernetes/tests/features/steps/minifi_as_pod_in_kubernetes_cluster.py +++ b/extensions/kubernetes/tests/features/steps/minifi_as_pod_in_kubernetes_cluster.py @@ -18,6 +18,8 @@ from minifi_behave.containers.minifi_container import MinifiContainer from minifi_behave.core.minifi_test_context import MinifiTestContext +logger = logging.getLogger(__name__) + class MinifiAsPodInKubernetesCluster(MinifiContainer): def __init__(self, container_name: str, test_context: MinifiTestContext): @@ -25,7 +27,7 @@ def __init__(self, container_name: str, test_context: MinifiTestContext): self.container = test_context.kubernetes_proxy def deploy(self, context: MinifiTestContext | None) -> bool: - logging.debug('Setting up the kind Kubernetes cluster') + logger.debug("Setting up the kind Kubernetes cluster") self.container.write_minifi_conf_file("minifi.properties", self._get_properties_file_content()) self.container.write_minifi_conf_file("minifi-log.properties", self._get_log_properties_file_content()) self.container.write_minifi_conf_file("config.yml", self.flow_definition.to_yaml()) diff --git a/extensions/kubernetes/tests/features/steps/steps.py b/extensions/kubernetes/tests/features/steps/steps.py index b153cadb07..a41d18e73b 100644 --- a/extensions/kubernetes/tests/features/steps/steps.py +++ b/extensions/kubernetes/tests/features/steps/steps.py @@ -14,20 +14,26 @@ # limitations under the License. from behave import given - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 - -from minifi_behave.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext -from minifi_behave.minifi.processor import Processor -from minifi_behave.minifi.controller_service import ControllerService from minifi_as_pod_in_kubernetes_cluster import MinifiAsPodInKubernetesCluster +from minifi_behave.core.minifi_test_context import ( + DEFAULT_MINIFI_CONTAINER_NAME, + MinifiTestContext, +) +from minifi_behave.minifi.controller_service import ControllerService +from minifi_behave.minifi.processor import Processor +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) def __ensure_kubernetes_cluster(context: MinifiTestContext): - if DEFAULT_MINIFI_CONTAINER_NAME not in context.containers or not isinstance(context.containers[DEFAULT_MINIFI_CONTAINER_NAME], MinifiAsPodInKubernetesCluster): + if DEFAULT_MINIFI_CONTAINER_NAME not in context.containers or not isinstance( + context.containers[DEFAULT_MINIFI_CONTAINER_NAME], + MinifiAsPodInKubernetesCluster, + ): context.containers[DEFAULT_MINIFI_CONTAINER_NAME] = MinifiAsPodInKubernetesCluster("kubernetes", context) @@ -39,8 +45,16 @@ def setup_processor_in_kubernetes_cluster(context: MinifiTestContext, processor_ context.get_or_create_default_minifi_container().flow_definition.add_processor(processor) -def __set_up_the_kubernetes_controller_service(context: MinifiTestContext, processor_name: str, service_property_name: str, properties: dict[str, str]): - kubernetes_controller_service = ControllerService(class_name="KubernetesControllerService", service_name="Kubernetes Controller Service") +def __set_up_the_kubernetes_controller_service( + context: MinifiTestContext, + processor_name: str, + service_property_name: str, + properties: dict[str, str], +): + kubernetes_controller_service = ControllerService( + class_name="KubernetesControllerService", + service_name="Kubernetes Controller Service", + ) kubernetes_controller_service.properties = properties flow = context.get_or_create_default_minifi_container().flow_definition flow.controller_services.append(kubernetes_controller_service) @@ -49,11 +63,25 @@ def __set_up_the_kubernetes_controller_service(context: MinifiTestContext, proce @given("the {processor_name} processor has a {service_property_name} which is a Kubernetes Controller Service") @given("the {processor_name} processor has an {service_property_name} which is a Kubernetes Controller Service") -def setup_kubernetes_controller_service_for_processor(context: MinifiTestContext, processor_name: str, service_property_name: str): +def setup_kubernetes_controller_service_for_processor( + context: MinifiTestContext, processor_name: str, service_property_name: str +): __set_up_the_kubernetes_controller_service(context, processor_name, service_property_name, {}) -@given("the {processor_name} processor has a {service_property_name} which is a Kubernetes Controller Service with the \"{property_name}\" property set to \"{property_value}\"") -@given("the {processor_name} processor has an {service_property_name} which is a Kubernetes Controller Service with the \"{property_name}\" property set to \"{property_value}\"") -def setup_kubernetes_controller_service_with_property_for_processor(context: MinifiTestContext, processor_name: str, service_property_name: str, property_name: str, property_value: str): - __set_up_the_kubernetes_controller_service(context, processor_name, service_property_name, {property_name: property_value}) +@given( + 'the {processor_name} processor has a {service_property_name} which is a Kubernetes Controller Service with the "{property_name}" property set to "{property_value}"' +) +@given( + 'the {processor_name} processor has an {service_property_name} which is a Kubernetes Controller Service with the "{property_name}" property set to "{property_value}"' +) +def setup_kubernetes_controller_service_with_property_for_processor( + context: MinifiTestContext, + processor_name: str, + service_property_name: str, + property_name: str, + property_value: str, +): + __set_up_the_kubernetes_controller_service( + context, processor_name, service_property_name, {property_name: property_value} + ) diff --git a/extensions/llamacpp/tests/features/environment.py b/extensions/llamacpp/tests/features/environment.py index d89a597ae6..59592b34cd 100644 --- a/extensions/llamacpp/tests/features/environment.py +++ b/extensions/llamacpp/tests/features/environment.py @@ -12,13 +12,15 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from pathlib import Path from textwrap import dedent -from pathlib import Path from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario -from minifi_behave.core.hooks import get_minifi_container_image +from minifi_behave.core.hooks import ( + common_after_scenario, + common_before_scenario, + get_minifi_container_image, +) from minifi_behave.core.minifi_test_context import MinifiTestContext # These hooks are executed by behave before and after each scenario @@ -33,7 +35,8 @@ def before_all(context: MinifiTestContext): with open(wget_with_retry_path, "rb") as f: wget_with_retry_content = f.read() - dockerfile = dedent("""\ + dockerfile = dedent( + """\ FROM {base_image} USER root COPY wget_with_retry.sh /scripts/wget_with_retry.sh @@ -41,12 +44,13 @@ def before_all(context: MinifiTestContext): /scripts/wget_with_retry.sh https://huggingface.co/bartowski/Qwen2-VL-2B-Instruct-GGUF/resolve/main/Qwen2-VL-2B-Instruct-Q3_K_M.gguf {models_path} && \\ /scripts/wget_with_retry.sh https://huggingface.co/bartowski/Qwen2-VL-2B-Instruct-GGUF/resolve/main/mmproj-Qwen2-VL-2B-Instruct-f16.gguf {models_path} USER minificpp - """.format(base_image=minifi_container_image, models_path='/tmp/models')) + """.format(base_image=minifi_container_image, models_path="/tmp/models") + ) builder = DockerImageBuilder( image_tag="apacheminificpp:llama", dockerfile_content=dockerfile, - files_on_context={"wget_with_retry.sh": wget_with_retry_content} + files_on_context={"wget_with_retry.sh": wget_with_retry_content}, ) builder.build() diff --git a/extensions/llamacpp/tests/features/steps/steps.py b/extensions/llamacpp/tests/features/steps/steps.py index 25759880fa..ff5817598b 100644 --- a/extensions/llamacpp/tests/features/steps/steps.py +++ b/extensions/llamacpp/tests/features/steps/steps.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) diff --git a/extensions/lua/tests/features/environment.py b/extensions/lua/tests/features/environment.py index 7e9fcda289..6d98b376da 100644 --- a/extensions/lua/tests/features/environment.py +++ b/extensions/lua/tests/features/environment.py @@ -15,13 +15,12 @@ import os -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario def before_scenario(context, scenario): common_before_scenario(context, scenario) - context.resource_dir = os.path.join(os.path.dirname(__file__), 'resources') + context.resource_dir = os.path.join(os.path.dirname(__file__), "resources") def after_scenario(context, scenario): diff --git a/extensions/lua/tests/features/steps/steps.py b/extensions/lua/tests/features/steps/steps.py index 25759880fa..ff5817598b 100644 --- a/extensions/lua/tests/features/steps/steps.py +++ b/extensions/lua/tests/features/steps/steps.py @@ -13,7 +13,9 @@ # See the License for the specific language governing permissions and # limitations under the License. -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) diff --git a/extensions/mqtt/tests/features/containers/mqtt_broker_container.py b/extensions/mqtt/tests/features/containers/mqtt_broker_container.py index 8d38663a54..156834f26d 100644 --- a/extensions/mqtt/tests/features/containers/mqtt_broker_container.py +++ b/extensions/mqtt/tests/features/containers/mqtt_broker_container.py @@ -17,30 +17,35 @@ import re from textwrap import dedent +from docker.errors import ContainerError from minifi_behave.containers.container_linux import LinuxContainer +from minifi_behave.containers.docker_image_builder import DockerImageBuilder from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from docker.errors import ContainerError + +logger = logging.getLogger(__name__) class MqttBrokerContainer(LinuxContainer): IMAGE = "eclipse-mosquitto:2.1.1-alpine" def __init__(self, test_context: MinifiTestContext): - dockerfile = dedent("""\ - FROM {base_image} + dockerfile = dedent( + f"""\ + FROM {MqttBrokerContainer.IMAGE} RUN echo 'log_dest stderr' >> /mosquitto-no-auth.conf CMD ["/usr/sbin/mosquitto", "--verbose", "--config-file", "/mosquitto-no-auth.conf"] - """.format(base_image=MqttBrokerContainer.IMAGE)) - - builder = DockerImageBuilder( - image_tag="minifi-mqtt-broker:latest", - dockerfile_content=dockerfile + """ ) + + builder = DockerImageBuilder(image_tag="minifi-mqtt-broker:latest", dockerfile_content=dockerfile) builder.build() - super().__init__("minifi-mqtt-broker:latest", f"mqtt-broker-{test_context.scenario_id}", test_context.network) + super().__init__( + "minifi-mqtt-broker:latest", + f"mqtt-broker-{test_context.scenario_id}", + test_context.network, + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -49,17 +54,33 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: re.search(finished_str, self.get_logs()), timeout_seconds=60, bail_condition=lambda: self.exited, - context=context) + context=context, + ) def publish_mqtt_message(self, topic: str, message: str): try: - self.client.containers.run("minifi-mqtt-helper:latest", ["python", "/scripts/publish_mqtt_message.py", self.container_name, topic, message], remove=True, stdout=True, stderr=True, network=self.network.name) + self.client.containers.run( + "minifi-mqtt-helper:latest", + [ + "python", + "/scripts/publish_mqtt_message.py", + self.container_name, + topic, + message, + ], + remove=True, + stdout=True, + stderr=True, + network=self.network.name, + ) return True except ContainerError as e: stdout = e.stdout.decode("utf-8", errors="replace") if hasattr(e, "stdout") and e.stdout else "" stderr = e.stderr.decode("utf-8", errors="replace") if hasattr(e, "stderr") and e.stderr else "" - logging.error(f"Failed to publish mqtt message in mqtt helper docker with error: '{e}', stdout: '{stdout}', stderr: '{stderr}'") + logger.error( + f"Failed to publish mqtt message in mqtt helper docker with error: '{e}', stdout: '{stdout}', stderr: '{stderr}'" + ) return False except Exception as e: - logging.error(f"Unexpected error while publishing mqtt message in mqtt helper docker: '{e}'") + logger.error(f"Unexpected error while publishing mqtt message in mqtt helper docker: '{e}'") return False diff --git a/extensions/mqtt/tests/features/environment.py b/extensions/mqtt/tests/features/environment.py index 6cabeecf03..eaee5a7c2b 100644 --- a/extensions/mqtt/tests/features/environment.py +++ b/extensions/mqtt/tests/features/environment.py @@ -13,12 +13,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import docker -from containers.mqtt_broker_container import MqttBrokerContainer from pathlib import Path + +from containers.mqtt_broker_container import MqttBrokerContainer from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_all(context): @@ -33,7 +34,7 @@ def before_all(context): mqtt_helper_builder = DockerImageBuilder( image_tag="minifi-mqtt-helper:latest", dockerfile_content=dockerfile, - files_on_context={"publish_mqtt_message.py": check_log_lines_content} + files_on_context={"publish_mqtt_message.py": check_log_lines_content}, ) mqtt_helper_builder.build() docker.from_env().images.pull(MqttBrokerContainer.IMAGE) diff --git a/extensions/mqtt/tests/features/resources/publish_mqtt_message.py b/extensions/mqtt/tests/features/resources/publish_mqtt_message.py index 6fe709acf5..88cb17dda4 100644 --- a/extensions/mqtt/tests/features/resources/publish_mqtt_message.py +++ b/extensions/mqtt/tests/features/resources/publish_mqtt_message.py @@ -12,9 +12,10 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import paho.mqtt.client as mqtt import sys +import paho.mqtt.client as mqtt + def publish_test_mqtt_message(host: str, topic: str, message: str): client = mqtt.Client(mqtt.CallbackAPIVersion.VERSION2, "docker_test_client_id") diff --git a/extensions/mqtt/tests/features/steps/steps.py b/extensions/mqtt/tests/features/steps/steps.py index 1bce835ae5..2f98d7a880 100644 --- a/extensions/mqtt/tests/features/steps/steps.py +++ b/extensions/mqtt/tests/features/steps/steps.py @@ -14,28 +14,32 @@ # limitations under the License. import re -from behave import given, step, then, when - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.core.minifi_test_context import DEFAULT_MINIFI_CONTAINER_NAME, MinifiTestContext -from minifi_behave.minifi.processor import Processor -from minifi_behave.core.helpers import wait_for_condition +from behave import given, step, then, when from containers.mqtt_broker_container import MqttBrokerContainer +from minifi_behave.core.helpers import wait_for_condition +from minifi_behave.core.minifi_test_context import ( + DEFAULT_MINIFI_CONTAINER_NAME, + MinifiTestContext, +) +from minifi_behave.minifi.processor import Processor +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) -@given("a {processor_name} processor set up to communicate with an MQTT broker instance in the \"{container_name}\" flow") +@given('a {processor_name} processor set up to communicate with an MQTT broker instance in the "{container_name}" flow') def setup_mqtt_processor_in_flow(context: MinifiTestContext, processor_name: str, container_name: str): processor = Processor(processor_name, processor_name) - processor.add_property('Broker URI', f'mqtt-broker-{context.scenario_id}:1883') - processor.add_property('Topic', 'testtopic') - if processor_name == 'PublishMQTT': - processor.add_property('Client ID', 'publisher-client') - elif processor_name == 'ConsumeMQTT': - processor.add_property('Client ID', 'consumer-client') + processor.add_property("Broker URI", f"mqtt-broker-{context.scenario_id}:1883") + processor.add_property("Topic", "testtopic") + if processor_name == "PublishMQTT": + processor.add_property("Client ID", "publisher-client") + elif processor_name == "ConsumeMQTT": + processor.add_property("Client ID", "consumer-client") else: raise ValueError(f"Unknown processor to communicate with MQTT broker: {processor_name}") @@ -44,7 +48,9 @@ def setup_mqtt_processor_in_flow(context: MinifiTestContext, processor_name: str @given("a {processor_name} processor set up to communicate with an MQTT broker instance") def setup_mqtt_processor(context: MinifiTestContext, processor_name: str): - context.execute_steps(f'given a {processor_name} processor set up to communicate with an MQTT broker instance in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow') + context.execute_steps( + f'given a {processor_name} processor set up to communicate with an MQTT broker instance in the "{DEFAULT_MINIFI_CONTAINER_NAME}" flow' + ) @step("an MQTT broker is started") @@ -59,7 +65,8 @@ def verify_mqtt_broker_log_line_matches(context: MinifiTestContext, log_line_reg condition=lambda: re.search(log_line_regex, context.containers["mqtt-broker"].get_logs()), timeout_seconds=60, bail_condition=lambda: context.containers["mqtt-broker"].exited, - context=None) + context=None, + ) @then('the MQTT broker has {log_count:d} log lines matching "{log_line_regex}"') @@ -68,9 +75,10 @@ def verify_mqtt_broker_log_line_count_matches(context: MinifiTestContext, log_co condition=lambda: len(re.findall(log_line_regex, context.containers["mqtt-broker"].get_logs())) == log_count, timeout_seconds=60, bail_condition=lambda: context.containers["mqtt-broker"].exited, - context=None) + context=None, + ) -@when("a test message \"{message}\" is published to the MQTT broker on topic \"{topic}\"") +@when('a test message "{message}" is published to the MQTT broker on topic "{topic}"') def publish_test_message_to_mqtt_broker(context: MinifiTestContext, message: str, topic: str): assert context.containers["mqtt-broker"].publish_mqtt_message(topic, message) diff --git a/extensions/opc/tests/features/containers/opc_ua_server_container.py b/extensions/opc/tests/features/containers/opc_ua_server_container.py index 5bb8a233bc..c1c55df740 100644 --- a/extensions/opc/tests/features/containers/opc_ua_server_container.py +++ b/extensions/opc/tests/features/containers/opc_ua_server_container.py @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import List, Optional + from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext @@ -22,8 +22,13 @@ class OPCUAServerContainer(LinuxContainer): IMAGE = "lordgamez/open62541:1.5.2" - def __init__(self, test_context: MinifiTestContext, command: Optional[List[str]] = None): - super().__init__(OPCUAServerContainer.IMAGE, f"opcua-server-{test_context.scenario_id}", test_context.network, command=command) + def __init__(self, test_context: MinifiTestContext, command: list[str] | None = None): + super().__init__( + OPCUAServerContainer.IMAGE, + f"opcua-server-{test_context.scenario_id}", + test_context.network, + command=command, + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -32,4 +37,5 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=30, bail_condition=lambda: self.exited, - context=context) + context=context, + ) diff --git a/extensions/opc/tests/features/environment.py b/extensions/opc/tests/features/environment.py index 76f86f8b86..0f9837f491 100644 --- a/extensions/opc/tests/features/environment.py +++ b/extensions/opc/tests/features/environment.py @@ -14,10 +14,11 @@ # limitations under the License. import os -import docker + from containers.opc_ua_server_container import OPCUAServerContainer -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_all(context): @@ -26,7 +27,7 @@ def before_all(context): def before_scenario(context, scenario): common_before_scenario(context, scenario) - context.resource_dir = os.path.join(os.path.dirname(__file__), 'resources') + context.resource_dir = os.path.join(os.path.dirname(__file__), "resources") def after_scenario(context, scenario): diff --git a/extensions/opc/tests/features/steps/steps.py b/extensions/opc/tests/features/steps/steps.py index 58f2a1ea1b..7e39976ad6 100644 --- a/extensions/opc/tests/features/steps/steps.py +++ b/extensions/opc/tests/features/steps/steps.py @@ -13,22 +13,27 @@ # See the License for the specific language governing permissions and # limitations under the License. +import io import logging import os -import tempfile -import docker -import io import tarfile -import humanfriendly +import tempfile +import humanfriendly from behave import given, step, then -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.helpers import wait_for_condition from containers.opc_ua_server_container import OPCUAServerContainer +from minifi_behave.core.helpers import wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) + +import docker + +logger = logging.getLogger(__name__) @step("an OPC UA server is set up") @@ -38,10 +43,12 @@ def setup_opcua_server(context: MinifiTestContext): @step("an OPC UA server is set up with access control") def setup_opcua_server_with_access_control(context: MinifiTestContext): - context.containers["opcua-server-access"] = OPCUAServerContainer(context, command=["/opt/open62541/examples/access_control_server"]) + context.containers["opcua-server-access"] = OPCUAServerContainer( + context, command=["/opt/open62541/examples/access_control_server"] + ) -@then("the OPC UA server logs contain the following message: \"{log_message}\" in less than {duration}") +@then('the OPC UA server logs contain the following message: "{log_message}" in less than {duration}') def verify_opcua_server_logs_contain_message(context, log_message, duration): timeout_seconds = humanfriendly.parse_timespan(duration) opcua_container = context.containers["opcua-server"] @@ -50,7 +57,8 @@ def verify_opcua_server_logs_contain_message(context, log_message, duration): condition=lambda: log_message in opcua_container.get_logs(), timeout_seconds=timeout_seconds, bail_condition=lambda: opcua_container.exited, - context=context) + context=context, + ) def _copy_file_from_docker_image(image_name: str, file_path: str, output_path: str): @@ -69,18 +77,34 @@ def _copy_file_from_docker_image(image_name: str, file_path: str, output_path: s return True except Exception as e: - logging.error(f"Error copying file {file_path} from Docker image {image_name}: {e}") + logger.error(f"Error copying file {file_path} from Docker image {image_name}: {e}") return False finally: container.remove(force=True) -@given('the OPC UA server certificate files are placed in the "{directory}" directory in the MiNiFi container "{container_name}"') +@given( + 'the OPC UA server certificate files are placed in the "{directory}" directory in the MiNiFi container "{container_name}"' +) def place_opcua_certificate_files_in_minifi_container(context: MinifiTestContext, directory: str, container_name: str): if not hasattr(context, "opcua_cert_temp_dir"): context.opcua_cert_temp_dir = tempfile.TemporaryDirectory() - _copy_file_from_docker_image(OPCUAServerContainer.IMAGE, "/opt/open62541/pki/created/server_cert.der", os.path.join(context.opcua_cert_temp_dir.name, "server_cert.der")) - _copy_file_from_docker_image(OPCUAServerContainer.IMAGE, "/opt/open62541/pki/created/server_key.der", os.path.join(context.opcua_cert_temp_dir.name, "server_key.der")) - - context.get_or_create_minifi_container(container_name).add_host_file(os.path.join(context.opcua_cert_temp_dir.name, "server_cert.der"), os.path.join(directory, "opcua_client_cert.der")) - context.get_or_create_minifi_container(container_name).add_host_file(os.path.join(context.opcua_cert_temp_dir.name, "server_key.der"), os.path.join(directory, "opcua_client_key.der")) + _copy_file_from_docker_image( + OPCUAServerContainer.IMAGE, + "/opt/open62541/pki/created/server_cert.der", + os.path.join(context.opcua_cert_temp_dir.name, "server_cert.der"), + ) + _copy_file_from_docker_image( + OPCUAServerContainer.IMAGE, + "/opt/open62541/pki/created/server_key.der", + os.path.join(context.opcua_cert_temp_dir.name, "server_key.der"), + ) + + context.get_or_create_minifi_container(container_name).add_host_file( + os.path.join(context.opcua_cert_temp_dir.name, "server_cert.der"), + os.path.join(directory, "opcua_client_cert.der"), + ) + context.get_or_create_minifi_container(container_name).add_host_file( + os.path.join(context.opcua_cert_temp_dir.name, "server_key.der"), + os.path.join(directory, "opcua_client_key.der"), + ) diff --git a/extensions/prometheus/tests/features/containers/prometheus_container.py b/extensions/prometheus/tests/features/containers/prometheus_container.py index ee962d8ebf..32d3c53b19 100644 --- a/extensions/prometheus/tests/features/containers/prometheus_container.py +++ b/extensions/prometheus/tests/features/containers/prometheus_container.py @@ -14,50 +14,86 @@ # limitations under the License. import logging + +from minifi_behave.containers.container_linux import LinuxContainer +from minifi_behave.containers.file import File from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.ssl_utils import make_cert_without_extended_usage -from minifi_behave.containers.file import File -from minifi_behave.core.ssl_utils import dump_cert, dump_key -from minifi_behave.containers.container_linux import LinuxContainer +from minifi_behave.core.ssl_utils import ( + dump_cert, + dump_key, + make_cert_without_extended_usage, +) + +logger = logging.getLogger(__name__) class PrometheusContainer(LinuxContainer): IMAGE = "prom/prometheus:v3.9.1" def __init__(self, test_context: MinifiTestContext, ssl: bool = False): - super().__init__(PrometheusContainer.IMAGE, f"prometheus-{test_context.scenario_id}", test_context.network) + super().__init__( + PrometheusContainer.IMAGE, + f"prometheus-{test_context.scenario_id}", + test_context.network, + ) self.scenario_id = test_context.scenario_id extra_ssl_settings = "" if ssl: - prometheus_cert, prometheus_key = make_cert_without_extended_usage(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + prometheus_cert, prometheus_key = make_cert_without_extended_usage( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) root_ca_content = dump_cert(test_context.root_ca_cert) - self.files.append(File("/etc/prometheus/certs/root-ca.pem", root_ca_content, permissions=0o644)) + self.files.append( + File( + "/etc/prometheus/certs/root-ca.pem", + root_ca_content, + permissions=0o644, + ) + ) prometheus_cert_content = dump_cert(prometheus_cert) - self.files.append(File("/etc/prometheus/certs/prometheus.crt", prometheus_cert_content, permissions=0o644)) + self.files.append( + File( + "/etc/prometheus/certs/prometheus.crt", + prometheus_cert_content, + permissions=0o644, + ) + ) prometheus_key_content = dump_key(prometheus_key) - self.files.append(File("/etc/prometheus/certs/prometheus.key", prometheus_key_content, permissions=0o644)) + self.files.append( + File( + "/etc/prometheus/certs/prometheus.key", + prometheus_key_content, + permissions=0o644, + ) + ) extra_ssl_settings = """ scheme: https tls_config: ca_file: /etc/prometheus/certs/root-ca.pem """ - prometheus_yml_content = """ + prometheus_yml_content = f""" global: scrape_interval: 2s evaluation_interval: 15s scrape_configs: - job_name: "minifi" static_configs: - - targets: ["minifi-primary-{scenario_id}:9936"] + - targets: ["minifi-primary-{test_context.scenario_id}:9936"] {extra_ssl_settings} -""".format(scenario_id=test_context.scenario_id, extra_ssl_settings=extra_ssl_settings) +""" - self.files.append(File("/etc/prometheus/prometheus.yml", prometheus_yml_content, permissions=0o666)) + self.files.append( + File( + "/etc/prometheus/prometheus.yml", + prometheus_yml_content, + permissions=0o666, + ) + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -66,29 +102,67 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=60, bail_condition=lambda: self.exited, - context=context + context=context, ) def check_metric_class_on_prometheus(self, metric_class: str) -> bool: try: - self.client.containers.run("minifi-prometheus-helper:latest", ["python", "/scripts/prometheus_checker.py", "--prometheus-host", self.container_name, "--metric-class", metric_class], remove=True, network=self.network.name) + self.client.containers.run( + "minifi-prometheus-helper:latest", + [ + "python", + "/scripts/prometheus_checker.py", + "--prometheus-host", + self.container_name, + "--metric-class", + metric_class, + ], + remove=True, + network=self.network.name, + ) return True except Exception: - logging.error(f"Failed to check metric class {metric_class} on Prometheus") + logger.error(f"Failed to check metric class {metric_class} on Prometheus") return False def check_processor_metric_on_prometheus(self, metric_class: str, processor_name: str) -> bool: try: - self.client.containers.run("minifi-prometheus-helper:latest", ["python", "/scripts/prometheus_checker.py", "--prometheus-host", self.container_name, "--metric-class", metric_class, "--processor-name", processor_name], remove=True, network=self.network.name) + self.client.containers.run( + "minifi-prometheus-helper:latest", + [ + "python", + "/scripts/prometheus_checker.py", + "--prometheus-host", + self.container_name, + "--metric-class", + metric_class, + "--processor-name", + processor_name, + ], + remove=True, + network=self.network.name, + ) return True except Exception: - logging.error(f"Failed to check processor metric class {metric_class} for processor {processor_name} on Prometheus") + logger.error( + f"Failed to check processor metric class {metric_class} for processor {processor_name} on Prometheus" + ) return False def check_all_metric_types_defined_once(self) -> bool: try: - self.client.containers.run("minifi-prometheus-helper:latest", ["python", "/scripts/prometheus_checker.py", "--verify-metrics-defined-once", f"minifi-primary-{self.scenario_id}"], remove=True, network=self.network.name) + self.client.containers.run( + "minifi-prometheus-helper:latest", + [ + "python", + "/scripts/prometheus_checker.py", + "--verify-metrics-defined-once", + f"minifi-primary-{self.scenario_id}", + ], + remove=True, + network=self.network.name, + ) return True except Exception: - logging.error("Failed check that all metric types are defined once on Prometheus") + logger.error("Failed check that all metric types are defined once on Prometheus") return False diff --git a/extensions/prometheus/tests/features/environment.py b/extensions/prometheus/tests/features/environment.py index c62553bdf7..859069e0b7 100644 --- a/extensions/prometheus/tests/features/environment.py +++ b/extensions/prometheus/tests/features/environment.py @@ -12,12 +12,13 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import docker from pathlib import Path -from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario + from containers.prometheus_container import PrometheusContainer +from minifi_behave.containers.docker_image_builder import DockerImageBuilder +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_all(context): @@ -32,7 +33,7 @@ def before_all(context): prometheus_helper_builder = DockerImageBuilder( image_tag="minifi-prometheus-helper:latest", dockerfile_content=dockerfile, - files_on_context={"prometheus_checker.py": check_log_lines_content} + files_on_context={"prometheus_checker.py": check_log_lines_content}, ) prometheus_helper_builder.build() docker.from_env().images.pull(PrometheusContainer.IMAGE) diff --git a/extensions/prometheus/tests/features/resources/prometheus_checker.py b/extensions/prometheus/tests/features/resources/prometheus_checker.py index e37662890c..7c63d8627a 100644 --- a/extensions/prometheus/tests/features/resources/prometheus_checker.py +++ b/extensions/prometheus/tests/features/resources/prometheus_checker.py @@ -14,6 +14,7 @@ # limitations under the License. import argparse import sys + import requests from prometheus_api_client import PrometheusConnect @@ -40,80 +41,190 @@ def verify_metric_class(self, metric_class: str) -> bool: elif metric_class == "AgentStatus": return self._verify_agent_status_metrics() else: - raise Exception("Metric class '%s' verification is not implemented" % metric_class) + raise RuntimeError(f"Metric class '{metric_class}' verification is not implemented") def _verify_repository_metrics(self) -> bool: - label_list = [{'repository_name': 'provenance'}, {'repository_name': 'flowfile'}, {'repository_name': 'content'}] + label_list = [ + {"repository_name": "provenance"}, + {"repository_name": "flowfile"}, + {"repository_name": "content"}, + ] # Only flowfile and content repositories are using rocksdb by default, so rocksdb specific metrics are only present there - return all((self._verify_metrics_exist(['minifi_is_running', 'minifi_is_full', 'minifi_repository_size_bytes', 'minifi_max_repository_size_bytes', 'minifi_repository_entry_count'], 'RepositoryMetrics', labels) for labels in label_list)) and \ - all((self._verify_metric_larger_than_zero('minifi_repository_size_bytes', 'RepositoryMetrics', labels) for labels in label_list[1:3])) and \ - all((self._verify_metrics_exist(['minifi_rocksdb_table_readers_size_bytes', 'minifi_rocksdb_all_memory_tables_size_bytes'], 'RepositoryMetrics', labels) for labels in label_list[1:3])) + return ( + all( + self._verify_metrics_exist( + [ + "minifi_is_running", + "minifi_is_full", + "minifi_repository_size_bytes", + "minifi_max_repository_size_bytes", + "minifi_repository_entry_count", + ], + "RepositoryMetrics", + labels, + ) + for labels in label_list + ) + and all( + self._verify_metric_larger_than_zero("minifi_repository_size_bytes", "RepositoryMetrics", labels) + for labels in label_list[1:3] + ) + and all( + self._verify_metrics_exist( + [ + "minifi_rocksdb_table_readers_size_bytes", + "minifi_rocksdb_all_memory_tables_size_bytes", + ], + "RepositoryMetrics", + labels, + ) + for labels in label_list[1:3] + ) + ) def _verify_queue_metrics(self) -> bool: - return self._verify_metrics_exist(['minifi_queue_data_size', 'minifi_queue_data_size_max', 'minifi_queue_size', 'minifi_queue_size_max'], 'QueueMetrics') + return self._verify_metrics_exist( + [ + "minifi_queue_data_size", + "minifi_queue_data_size_max", + "minifi_queue_size", + "minifi_queue_size_max", + ], + "QueueMetrics", + ) def _verify_general_processor_metrics(self, metric_class: str, processor_name: str) -> bool: - labels = {'processor_name': processor_name} - return self._verify_metrics_exist(['minifi_average_onTrigger_runtime_milliseconds', 'minifi_last_onTrigger_runtime_milliseconds', - 'minifi_average_session_commit_runtime_milliseconds', 'minifi_last_session_commit_runtime_milliseconds', - 'minifi_incoming_flow_files', 'minifi_incoming_bytes', 'minifi_bytes_read', 'minifi_bytes_written'], metric_class, labels) and \ - self._verify_metrics_larger_than_zero(['minifi_onTrigger_invocations', 'minifi_transferred_flow_files', 'minifi_transferred_to_success', - 'minifi_transferred_bytes', 'minifi_processing_nanos'], - metric_class, labels) + labels = {"processor_name": processor_name} + return self._verify_metrics_exist( + [ + "minifi_average_onTrigger_runtime_milliseconds", + "minifi_last_onTrigger_runtime_milliseconds", + "minifi_average_session_commit_runtime_milliseconds", + "minifi_last_session_commit_runtime_milliseconds", + "minifi_incoming_flow_files", + "minifi_incoming_bytes", + "minifi_bytes_read", + "minifi_bytes_written", + ], + metric_class, + labels, + ) and self._verify_metrics_larger_than_zero( + [ + "minifi_onTrigger_invocations", + "minifi_transferred_flow_files", + "minifi_transferred_to_success", + "minifi_transferred_bytes", + "minifi_processing_nanos", + ], + metric_class, + labels, + ) def _verify_getfile_metrics(self, metric_class: str, processor_name: str) -> bool: - labels = {'processor_name': processor_name} - return self._verify_general_processor_metrics(metric_class, processor_name) and \ - self._verify_metrics_exist(['minifi_input_bytes', 'minifi_accepted_files'], metric_class, labels) + labels = {"processor_name": processor_name} + return self._verify_general_processor_metrics(metric_class, processor_name) and self._verify_metrics_exist( + ["minifi_input_bytes", "minifi_accepted_files"], metric_class, labels + ) def _verify_flow_information_metrics(self) -> bool: - return self._verify_metrics_exist(['minifi_queue_data_size', 'minifi_queue_data_size_max', 'minifi_queue_size', 'minifi_queue_size_max', - 'minifi_bytes_read', 'minifi_bytes_written', 'minifi_flow_files_in', 'minifi_flow_files_out', 'minifi_bytes_in', 'minifi_bytes_out', - 'minifi_invocations', 'minifi_processing_nanos'], 'FlowInformation') and \ - self._verify_metric_exists('minifi_is_running', 'FlowInformation', {'component_name': 'FlowController'}) + return self._verify_metrics_exist( + [ + "minifi_queue_data_size", + "minifi_queue_data_size_max", + "minifi_queue_size", + "minifi_queue_size_max", + "minifi_bytes_read", + "minifi_bytes_written", + "minifi_flow_files_in", + "minifi_flow_files_out", + "minifi_bytes_in", + "minifi_bytes_out", + "minifi_invocations", + "minifi_processing_nanos", + ], + "FlowInformation", + ) and self._verify_metric_exists("minifi_is_running", "FlowInformation", {"component_name": "FlowController"}) def _verify_device_info_node_metrics(self) -> bool: - return self._verify_metrics_exist(['minifi_physical_mem', 'minifi_memory_usage', 'minifi_cpu_utilization', 'minifi_cpu_load_average'], 'DeviceInfoNode') + return self._verify_metrics_exist( + [ + "minifi_physical_mem", + "minifi_memory_usage", + "minifi_cpu_utilization", + "minifi_cpu_load_average", + ], + "DeviceInfoNode", + ) def _verify_agent_status_metrics(self) -> bool: - label_list = [{'repository_name': 'flowfile'}, {'repository_name': 'content'}] + label_list = [{"repository_name": "flowfile"}, {"repository_name": "content"}] # Only flowfile and content repositories are using rocksdb by default, so rocksdb specific metrics are only present there for labels in label_list: - if not (self._verify_metric_exists('minifi_is_running', 'AgentStatus', labels) - and self._verify_metric_exists('minifi_is_full', 'AgentStatus', labels) - and self._verify_metric_exists('minifi_max_repository_size_bytes', 'AgentStatus', labels) - and self._verify_metric_larger_than_zero('minifi_repository_size_bytes', 'AgentStatus', labels) - and self._verify_metric_exists('minifi_repository_entry_count', 'AgentStatus', labels) - and self._verify_metric_exists('minifi_rocksdb_table_readers_size_bytes', 'AgentStatus', labels) - and self._verify_metric_exists('minifi_rocksdb_all_memory_tables_size_bytes', 'AgentStatus', labels)): + if not ( + self._verify_metric_exists("minifi_is_running", "AgentStatus", labels) + and self._verify_metric_exists("minifi_is_full", "AgentStatus", labels) + and self._verify_metric_exists("minifi_max_repository_size_bytes", "AgentStatus", labels) + and self._verify_metric_larger_than_zero("minifi_repository_size_bytes", "AgentStatus", labels) + and self._verify_metric_exists("minifi_repository_entry_count", "AgentStatus", labels) + and self._verify_metric_exists("minifi_rocksdb_table_readers_size_bytes", "AgentStatus", labels) + and self._verify_metric_exists("minifi_rocksdb_all_memory_tables_size_bytes", "AgentStatus", labels) + ): return False # provenance repository is NoOpRepository by default which has zero size - if not (self._verify_metric_exists('minifi_is_running', 'AgentStatus', {'repository_name': 'provenance'}) - and self._verify_metric_exists('minifi_is_full', 'AgentStatus', {'repository_name': 'provenance'}) - and self._verify_metric_exists('minifi_max_repository_size_bytes', 'AgentStatus', {'repository_name': 'provenance'}) - and self._verify_metric_exists('minifi_repository_size_bytes', 'AgentStatus', {'repository_name': 'provenance'}) - and self._verify_metric_exists('minifi_repository_entry_count', 'AgentStatus', {'repository_name': 'provenance'})): + if not ( + self._verify_metric_exists("minifi_is_running", "AgentStatus", {"repository_name": "provenance"}) + and self._verify_metric_exists("minifi_is_full", "AgentStatus", {"repository_name": "provenance"}) + and self._verify_metric_exists( + "minifi_max_repository_size_bytes", + "AgentStatus", + {"repository_name": "provenance"}, + ) + and self._verify_metric_exists( + "minifi_repository_size_bytes", + "AgentStatus", + {"repository_name": "provenance"}, + ) + and self._verify_metric_exists( + "minifi_repository_entry_count", + "AgentStatus", + {"repository_name": "provenance"}, + ) + ): return False - return self._verify_metric_exists('minifi_uptime_milliseconds', 'AgentStatus') and \ - self._verify_metric_exists('minifi_agent_memory_usage_bytes', 'AgentStatus') and \ - self._verify_metric_exists('minifi_agent_cpu_utilization', 'AgentStatus') + return ( + self._verify_metric_exists("minifi_uptime_milliseconds", "AgentStatus") + and self._verify_metric_exists("minifi_agent_memory_usage_bytes", "AgentStatus") + and self._verify_metric_exists("minifi_agent_cpu_utilization", "AgentStatus") + ) - def _verify_metric_exists(self, metric_name: str, metric_class: str, labels: dict = {}) -> bool: - labels['metric_class'] = metric_class - labels['agent_identifier'] = "Agent1" + def _verify_metric_exists(self, metric_name: str, metric_class: str, labels: dict | None = None) -> bool: + if labels is None: + labels = {} + labels["metric_class"] = metric_class + labels["agent_identifier"] = "Agent1" return len(self.prometheus_client.get_current_metric_value(metric_name=metric_name, label_config=labels)) > 0 - def _verify_metrics_exist(self, metric_names: list, metric_class: str, labels: dict = {}) -> bool: - return all((self._verify_metric_exists(metric_name, metric_class, labels) for metric_name in metric_names)) + def _verify_metrics_exist(self, metric_names: list, metric_class: str, labels: dict | None = None) -> bool: + if labels is None: + labels = {} + return all(self._verify_metric_exists(metric_name, metric_class, labels) for metric_name in metric_names) - def _verify_metric_larger_than_zero(self, metric_name: str, metric_class: str, labels: dict = {}) -> bool: - labels['metric_class'] = metric_class + def _verify_metric_larger_than_zero(self, metric_name: str, metric_class: str, labels: dict | None = None) -> bool: + if labels is None: + labels = {} + labels["metric_class"] = metric_class result = self.prometheus_client.get_current_metric_value(metric_name=metric_name, label_config=labels) - return len(result) > 0 and int(result[0]['value'][1]) > 0 + return len(result) > 0 and int(result[0]["value"][1]) > 0 - def _verify_metrics_larger_than_zero(self, metric_names: list, metric_class: str, labels: dict = {}) -> bool: - return all((self._verify_metric_larger_than_zero(metric_name, metric_class, labels) for metric_name in metric_names)) + def _verify_metrics_larger_than_zero( + self, metric_names: list, metric_class: str, labels: dict | None = None + ) -> bool: + if labels is None: + labels = {} + return all( + self._verify_metric_larger_than_zero(metric_name, metric_class, labels) for metric_name in metric_names + ) def verify_all_metric_types_are_defined_once(prometheus_target: str) -> bool: @@ -133,11 +244,16 @@ def verify_all_metric_types_are_defined_once(prometheus_target: str) -> bool: if __name__ == "__main__": - parser = argparse.ArgumentParser(description='Prometheus Checker') - parser.add_argument('--verify-metrics-defined-once', type=str, required=False, help='Prometheus target to verify all defined metrics are unique') - parser.add_argument('--prometheus-host', type=str, required=False, help='Prometheus server host') - parser.add_argument('--metric-class', type=str, required=False, help='Metric class to verify') - parser.add_argument('--processor-name', type=str, required=False, help='Processor name to verify') + parser = argparse.ArgumentParser(description="Prometheus Checker") + parser.add_argument( + "--verify-metrics-defined-once", + type=str, + required=False, + help="Prometheus target to verify all defined metrics are unique", + ) + parser.add_argument("--prometheus-host", type=str, required=False, help="Prometheus server host") + parser.add_argument("--metric-class", type=str, required=False, help="Metric class to verify") + parser.add_argument("--processor-name", type=str, required=False, help="Processor name to verify") args = parser.parse_args() diff --git a/extensions/prometheus/tests/features/steps/steps.py b/extensions/prometheus/tests/features/steps/steps.py index eb27aecb77..66f2fb7885 100644 --- a/extensions/prometheus/tests/features/steps/steps.py +++ b/extensions/prometheus/tests/features/steps/steps.py @@ -12,18 +12,19 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -from behave import step, then, given - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from behave import given, step, then +from containers.prometheus_container import PrometheusContainer from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext -from containers.prometheus_container import PrometheusContainer +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) -@step('a Prometheus server is set up') +@step("a Prometheus server is set up") def setup_prometheus_server(context: MinifiTestContext): context.containers["prometheus"] = PrometheusContainer(context) @@ -33,19 +34,34 @@ def setup_prometheus_server_with_ssl(context: MinifiTestContext): context.containers["prometheus"] = PrometheusContainer(context, ssl=True) -@then("\"{metric_class}\" are published to the Prometheus server in less than {timeout_seconds:d} seconds") -@then("\"{metric_class}\" is published to the Prometheus server in less than {timeout_seconds:d} seconds") +@then('"{metric_class}" are published to the Prometheus server in less than {timeout_seconds:d} seconds') +@then('"{metric_class}" is published to the Prometheus server in less than {timeout_seconds:d} seconds') def verify_metric_class_published_to_prometheus(context: MinifiTestContext, metric_class: str, timeout_seconds: int): assert wait_for_condition( condition=lambda: context.containers["prometheus"].check_metric_class_on_prometheus(metric_class), - timeout_seconds=timeout_seconds, bail_condition=lambda: context.containers["prometheus"].exited, context=context) - - -@then("\"{metric_class}\" processor metric is published to the Prometheus server in less than {timeout_seconds:d} seconds for \"{processor_name}\" processor") -def verify_processor_metric_published_to_prometheus(context: MinifiTestContext, metric_class: str, timeout_seconds: int, processor_name: str): + timeout_seconds=timeout_seconds, + bail_condition=lambda: context.containers["prometheus"].exited, + context=context, + ) + + +@then( + '"{metric_class}" processor metric is published to the Prometheus server in less than {timeout_seconds:d} seconds for "{processor_name}" processor' +) +def verify_processor_metric_published_to_prometheus( + context: MinifiTestContext, + metric_class: str, + timeout_seconds: int, + processor_name: str, +): assert wait_for_condition( - condition=lambda: context.containers["prometheus"].check_processor_metric_on_prometheus(metric_class, processor_name), - timeout_seconds=timeout_seconds, bail_condition=lambda: context.containers["prometheus"].exited, context=context) + condition=lambda: context.containers["prometheus"].check_processor_metric_on_prometheus( + metric_class, processor_name + ), + timeout_seconds=timeout_seconds, + bail_condition=lambda: context.containers["prometheus"].exited, + context=context, + ) @then("all Prometheus metric types are only defined once") @@ -55,10 +71,16 @@ def verify_all_prometheus_metric_types_defined_once(context: MinifiTestContext): def _enable_prometheus(context: MinifiTestContext): context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.agent.identifier", "Agent1") - context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.PrometheusMetricsPublisher.port", "9936") - context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.PrometheusMetricsPublisher.metrics", - "RepositoryMetrics,QueueMetrics,PutFileMetrics,processorMetrics/Get.*,FlowInformation,DeviceInfoNode,AgentStatus") - context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.class", "PrometheusMetricsPublisher") + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.PrometheusMetricsPublisher.port", "9936" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.PrometheusMetricsPublisher.metrics", + "RepositoryMetrics,QueueMetrics,PutFileMetrics,processorMetrics/Get.*,FlowInformation,DeviceInfoNode,AgentStatus", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.class", "PrometheusMetricsPublisher" + ) @given("Prometheus is enabled in MiNiFi") @@ -69,5 +91,11 @@ def enable_prometheus_in_minifi(context: MinifiTestContext): @given("Prometheus with SSL is enabled in MiNiFi") def enable_prometheus_with_ssl_in_minifi(context: MinifiTestContext): _enable_prometheus(context) - context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.PrometheusMetricsPublisher.certificate", "/tmp/resources/minifi_merged_cert.crt") - context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.PrometheusMetricsPublisher.ca.certificate", "/tmp/resources/root_ca.crt") + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.PrometheusMetricsPublisher.certificate", + "/tmp/resources/minifi_merged_cert.crt", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.metrics.publisher.PrometheusMetricsPublisher.ca.certificate", + "/tmp/resources/root_ca.crt", + ) diff --git a/extensions/python/CMakeLists.txt b/extensions/python/CMakeLists.txt index 34b1df6730..317bdeb881 100644 --- a/extensions/python/CMakeLists.txt +++ b/extensions/python/CMakeLists.txt @@ -38,14 +38,14 @@ target_link_libraries(minifi-python-script-extension PRIVATE ${LIBMINIFI} Thread include(GenericPython) if(APPLE) - target_compile_definitions(minifi-python-lib-loader-extension PUBLIC Py_LIMITED_API=0x03090000) - target_compile_definitions(minifi-python-script-extension PUBLIC Py_LIMITED_API=0x03090000) + target_compile_definitions(minifi-python-lib-loader-extension PUBLIC Py_LIMITED_API=0x030A0000) + target_compile_definitions(minifi-python-script-extension PUBLIC Py_LIMITED_API=0x030A0000) target_link_options(minifi-python-script-extension PRIVATE "-Wl,-undefined,dynamic_lookup") elseif(WIN32) - target_compile_definitions(minifi-python-script-extension PUBLIC Py_LIMITED_API=0x03060000) + target_compile_definitions(minifi-python-script-extension PUBLIC Py_LIMITED_API=0x030A0000) else() - target_compile_definitions(minifi-python-lib-loader-extension PUBLIC Py_LIMITED_API=0x03060000) - target_compile_definitions(minifi-python-script-extension PUBLIC Py_LIMITED_API=0x03060000) + target_compile_definitions(minifi-python-lib-loader-extension PUBLIC Py_LIMITED_API=0x030A0000) + target_compile_definitions(minifi-python-script-extension PUBLIC Py_LIMITED_API=0x030A0000) endif() target_compile_definitions(minifi-python-script-extension PUBLIC PY_SSIZE_T_CLEAN) diff --git a/extensions/python/PYTHON.md b/extensions/python/PYTHON.md index 66e7c29950..13fd38391d 100644 --- a/extensions/python/PYTHON.md +++ b/extensions/python/PYTHON.md @@ -180,7 +180,7 @@ Describe is passed the processor and is a required function. You must set the de ```python def describe(processor): - processor.setDescription("Adds an attribute to your flow files") + processor.setDescription("Adds an attribute to your flow files") ``` onInitialize is also passed the processor reference and can be where you set properties. The first argument is the property display name, @@ -200,9 +200,9 @@ The last parameter of addProperty is the controller service type. If the propert ```python def onInitialize(processor): - processor.setSupportsDynamicProperties() - # arguments: property name, description, default value, is required, expression language supported, is sensitive, property type code, controller service type name - processor.addProperty("property name", "description", "default value", True, False, False, 1, None) + processor.setSupportsDynamicProperties() + # arguments: property name, description, default value, is required, expression language supported, is sensitive, property type code, controller service type name + processor.addProperty("property name", "description", "default value", True, False, False, 1, None) ``` The onSchedule function is passed the context and session factory. This should be where your processor loads and reads properties via @@ -214,19 +214,19 @@ VaderSentiment ```python class VaderSentiment(object): - def __init__(self): - self.content = None + def __init__(self): + self.content = None - def process(self, input_stream): - self.content = codecs.getreader('utf-8')(input_stream).read() - return len(self.content) + def process(self, input_stream): + self.content = codecs.getreader("utf-8")(input_stream).read() + return len(self.content) ``` By default, the MiNiFi C++ style native python processors are executed on multiple threads in parallel if 'max concurrent tasks' is set to more than 1 in the flow configuration, but it is possible to set the processor to be single threaded by calling the `setSingleThreaded()` method in the processor while initializing. When a processor is set to be single threaded, only one thread will execute the processor, and setting 'max concurrent tasks' to more than 1 will not have any effect. ```python def onInitialize(processor): - processor.setSingleThreaded() + processor.setSingleThreaded() ``` ## Using NiFi Python Processors @@ -262,13 +262,13 @@ from nifiapi.decorators import trigger_serially @trigger_serially class SingleThreadedProcessor(FlowFileSource): class Java: - implements = ['org.apache.nifi.python.processor.FlowFileSource'] + implements = ["org.apache.nifi.python.processor.FlowFileSource"] def __init__(self, **kwargs): pass def create(self, context): - return FlowFileSourceResult(relationship='success', contents="result") + return FlowFileSourceResult(relationship="success", contents="result") ``` ## Use Python processors from virtualenv diff --git a/extensions/python/pythonprocessor-examples/AddPythonAttribute.py b/extensions/python/pythonprocessor-examples/AddPythonAttribute.py index 53c8a60009..c7944de14a 100644 --- a/extensions/python/pythonprocessor-examples/AddPythonAttribute.py +++ b/extensions/python/pythonprocessor-examples/AddPythonAttribute.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. diff --git a/extensions/python/pythonprocessor-examples/CountingProcessor.py b/extensions/python/pythonprocessor-examples/CountingProcessor.py index 024f62f305..1dac431cb7 100644 --- a/extensions/python/pythonprocessor-examples/CountingProcessor.py +++ b/extensions/python/pythonprocessor-examples/CountingProcessor.py @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -global i i = 0 @@ -22,7 +21,7 @@ def __init__(self, content): self.content = content def process(self, output_stream): - output_stream.write(self.content.encode('utf-8')) + output_stream.write(self.content.encode("utf-8")) return len(self.content) diff --git a/extensions/python/pythonprocessor-examples/GaussianDistributionWithNumpy.py b/extensions/python/pythonprocessor-examples/GaussianDistributionWithNumpy.py index 0ad00fce91..92e3a47365 100644 --- a/extensions/python/pythonprocessor-examples/GaussianDistributionWithNumpy.py +++ b/extensions/python/pythonprocessor-examples/GaussianDistributionWithNumpy.py @@ -21,7 +21,7 @@ def __init__(self, content): self.content = content def process(self, output_stream): - output_stream.write(self.content.encode('utf-8')) + output_stream.write(self.content.encode("utf-8")) return len(self.content) diff --git a/extensions/python/pythonprocessor-examples/MoveContentToJson.py b/extensions/python/pythonprocessor-examples/MoveContentToJson.py index 1b3068b07c..a27608ac9b 100644 --- a/extensions/python/pythonprocessor-examples/MoveContentToJson.py +++ b/extensions/python/pythonprocessor-examples/MoveContentToJson.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -19,7 +18,7 @@ class ReadCallback: def process(self, input_stream): - self.content = codecs.getreader('utf-8')(input_stream).read() + self.content = codecs.getreader("utf-8")(input_stream).read() return len(self.content) @@ -29,7 +28,7 @@ def __init__(self, content): def process(self, output_stream): json_content = json.dumps({"content": self.content}) - output_stream.write(json_content.encode('utf-8')) + output_stream.write(json_content.encode("utf-8")) return len(json_content) diff --git a/extensions/python/pythonprocessor-examples/RemoveFlowFile.py b/extensions/python/pythonprocessor-examples/RemoveFlowFile.py index 5235636b49..27ea7760fa 100644 --- a/extensions/python/pythonprocessor-examples/RemoveFlowFile.py +++ b/extensions/python/pythonprocessor-examples/RemoveFlowFile.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. diff --git a/extensions/python/pythonprocessor-examples/SentimentAnalysis.py b/extensions/python/pythonprocessor-examples/SentimentAnalysis.py index 4db22608d6..c3c433c2bc 100644 --- a/extensions/python/pythonprocessor-examples/SentimentAnalysis.py +++ b/extensions/python/pythonprocessor-examples/SentimentAnalysis.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -14,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. import codecs + from vaderSentiment.vaderSentiment import SentimentIntensityAnalyzer @@ -25,12 +25,12 @@ def onInitialize(processor): processor.setSupportsDynamicProperties() -class VaderSentiment(object): +class VaderSentiment: def __init__(self): self.content = None def process(self, input_stream): - self.content = codecs.getreader('utf-8')(input_stream).read() + self.content = codecs.getreader("utf-8")(input_stream).read() return len(self.content) @@ -41,7 +41,7 @@ def onTrigger(context, session): session.read(flow_file, sentiment) analyzer = SentimentIntensityAnalyzer() vs = analyzer.polarity_scores(sentiment.content) - flow_file.addAttribute("positive", str(vs['pos'])) - flow_file.addAttribute("negative", str(vs['neg'])) - flow_file.addAttribute("neutral", str(vs['neu'])) + flow_file.addAttribute("positive", str(vs["pos"])) + flow_file.addAttribute("negative", str(vs["neg"])) + flow_file.addAttribute("neutral", str(vs["neu"])) session.transfer(flow_file, REL_SUCCESS) diff --git a/extensions/python/pythonprocessor-examples/google/SentimentAnalyzer.py b/extensions/python/pythonprocessor-examples/google/SentimentAnalyzer.py index c7ab9c0a05..39dc9fcd4c 100644 --- a/extensions/python/pythonprocessor-examples/google/SentimentAnalyzer.py +++ b/extensions/python/pythonprocessor-examples/google/SentimentAnalyzer.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -14,18 +13,19 @@ # See the License for the specific language governing permissions and # limitations under the License. """ - Install the following with pip ( or pip3 ) +Install the following with pip ( or pip3 ) - pip install google-cloud-language +pip install google-cloud-language - -- the following were needed during development as we saw SSL timeout errors - pip install requests[security] - pip install -U httplib2 +-- the following were needed during development as we saw SSL timeout errors +pip install requests[security] +pip install -U httplib2 """ + import codecs + from google.cloud import language -from google.cloud.language import enums -from google.cloud.language import types +from google.cloud.language import enums, types def describe(processor): @@ -34,15 +34,21 @@ def describe(processor): def onInitialize(processor): # is required, - processor.addProperty("Credentials Path", "Path to your Google Credentials JSON File. Must exist on agent hosts.", "", True, False) + processor.addProperty( + "Credentials Path", + "Path to your Google Credentials JSON File. Must exist on agent hosts.", + "", + True, + False, + ) -class ContentExtract(object): +class ContentExtract: def __init__(self): self.content = None def process(self, input_stream): - self.content = codecs.getreader('utf-8')(input_stream).read() + self.content = codecs.getreader("utf-8")(input_stream).read() return len(self.content) diff --git a/extensions/python/pythonprocessor-examples/h2o/ConvertDsToCsv.py b/extensions/python/pythonprocessor-examples/h2o/ConvertDsToCsv.py index 566f4e3126..3cf9c6bf2a 100644 --- a/extensions/python/pythonprocessor-examples/h2o/ConvertDsToCsv.py +++ b/extensions/python/pythonprocessor-examples/h2o/ConvertDsToCsv.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -14,61 +13,61 @@ # See the License for the specific language governing permissions and # limitations under the License. """ - Install the following with pip +Install the following with pip - pip install datatable - pip install pandas +pip install datatable +pip install pandas """ + import codecs -import pandas as pd # noqa F401 -import datatable as dt from io import StringIO +import datatable as dt +import pandas as pd # noqa F401 + def describe(processor): - """ describe what this processor does - """ - processor.setDescription("Converts the data source content of incoming flow file to csv. It \ + """describe what this processor does""" + processor.setDescription( + "Converts the data source content of incoming flow file to csv. It \ supports a variety of data sources: pandas DataFrames, csv, numpy \ - arrays, dictionary, list, raw Python objects, etc") + arrays, dictionary, list, raw Python objects, etc" + ) def onInitialize(processor): - """ onInitialize is where you can set properties - """ + """onInitialize is where you can set properties""" processor.setSupportsDynamicProperties() -class ContentExtract(object): - """ ContentExtract callback class is defined for reading streams of data through the session - and has a process function that accepts the input stream +class ContentExtract: + """ContentExtract callback class is defined for reading streams of data through the session + and has a process function that accepts the input stream """ + def __init__(self): self.content = None def process(self, input_stream): - """ Use codecs getReader to read that data - """ - self.content = codecs.getreader('utf-8')(input_stream).read() + """Use codecs getReader to read that data""" + self.content = codecs.getreader("utf-8")(input_stream).read() return len(self.content) -class ContentWrite(object): - """ ContentWrite callback class is defined for writing streams of data through the session - """ +class ContentWrite: + """ContentWrite callback class is defined for writing streams of data through the session""" + def __init__(self, data): self.content = data def process(self, output_stream): - """ Use codecs getWriter to write data encoded to the stream - """ - codecs.getwriter('utf-8')(output_stream).write(self.content) + """Use codecs getWriter to write data encoded to the stream""" + codecs.getwriter("utf-8")(output_stream).write(self.content) return len(self.content) def onTrigger(context, session): - """ onTrigger is executed and passed processor context and session - """ + """onTrigger is executed and passed processor context and session""" flow_file = session.get() if flow_file is not None: read_cb = ContentExtract() diff --git a/extensions/python/pythonprocessor-examples/h2o/h2o3/mojo/ExecuteH2oMojoScoring.py b/extensions/python/pythonprocessor-examples/h2o/h2o3/mojo/ExecuteH2oMojoScoring.py index 0c6f09207a..e6a099eb12 100644 --- a/extensions/python/pythonprocessor-examples/h2o/h2o3/mojo/ExecuteH2oMojoScoring.py +++ b/extensions/python/pythonprocessor-examples/h2o/h2o3/mojo/ExecuteH2oMojoScoring.py @@ -1,4 +1,3 @@ -#!/usr/bin/env python # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -14,85 +13,117 @@ # See the License for the specific language governing permissions and # limitations under the License. """ - -- after downloading the mojo model from h2o3, the following packages - are needed to execute the model to do batch or real-time scoring +-- after downloading the mojo model from h2o3, the following packages + are needed to execute the model to do batch or real-time scoring - Make all packages available on your machine: +Make all packages available on your machine: - sudo apt-get -y update +sudo apt-get -y update - Install Java to include open source H2O-3 algorithms: +Install Java to include open source H2O-3 algorithms: - sudo apt-get -y install openjdk-8-jdk +sudo apt-get -y install openjdk-8-jdk - Install Datatable and pandas: +Install Datatable and pandas: - pip install datatable - pip install pandas +pip install datatable +pip install pandas - Option 1: Install H2O-3 with conda +Option 1: Install H2O-3 with conda - conda create -n h2o3-nifi-minifi python=3.6 - conda activate h2o3-nifi-minifi - conda config --append channels conda-forge - conda install -y -c h2oai h2o +conda create -n h2o3-nifi-minifi python=3.6 +conda activate h2o3-nifi-minifi +conda config --append channels conda-forge +conda install -y -c h2oai h2o - Option 2: Install H2O-3 with pip +Option 2: Install H2O-3 with pip - pip install requests - pip install tabulate - pip install "colorama>=0.3.8" - pip install future - pip uninstall h2o - If on Mac OS X, must include --user: - pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o --user - else: - pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o +pip install requests +pip install tabulate +pip install "colorama>=0.3.8" +pip install future +pip uninstall h2o +If on Mac OS X, must include --user: + pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o --user +else: + pip install -f http://h2o-release.s3.amazonaws.com/h2o/latest_stable_Py.html h2o """ -import h2o + import codecs -import pandas as pd # noqa: F401 + import datatable as dt +import h2o +import pandas as pd # noqa: F401 mojo_model = None def describe(processor): - """ describe what this processor does - """ - processor.setDescription("Executes H2O-3's MOJO Model in Python to do batch scoring or \ + """describe what this processor does""" + processor.setDescription( + "Executes H2O-3's MOJO Model in Python to do batch scoring or \ real-time scoring for one or more predicted label(s) on the tabular test data in \ the incoming flow file content. If tabular data is one row, then MOJO does real-time \ - scoring. If tabular data is multiple rows, then MOJO does batch scoring.") + scoring. If tabular data is multiple rows, then MOJO does batch scoring." + ) def onInitialize(processor): - """ onInitialize is where you can set properties - processor.addProperty(name, description, defaultValue, required, el) + """onInitialize is where you can set properties + processor.addProperty(name, description, defaultValue, required, el) """ - processor.addProperty("MOJO Model Filepath", "Add the filepath to the MOJO Model file. For example, \ - 'path/to/mojo-model/GBM_grid__1_AutoML_20200511_075150_model_180.zip'.", "", True, False) - - processor.addProperty("Is First Line Header", "Add True or False for whether first line is header.", - "True", True, False) - - processor.addProperty("Input Schema", "If first line is not header, then you must add Input Schema for \ + processor.addProperty( + "MOJO Model Filepath", + "Add the filepath to the MOJO Model file. For example, \ + 'path/to/mojo-model/GBM_grid__1_AutoML_20200511_075150_model_180.zip'.", + "", + True, + False, + ) + + processor.addProperty( + "Is First Line Header", + "Add True or False for whether first line is header.", + "True", + True, + False, + ) + + processor.addProperty( + "Input Schema", + "If first line is not header, then you must add Input Schema for \ incoming data.If there is more than one column name, write a comma separated list of \ - column names. Else, you do not need to add an Input Schema.", "", False, False) - - processor.addProperty("Use Output Header", "Add True or False for whether you want to use an output \ - for your predictions.", "False", False, False) - - processor.addProperty("Output Schema", "To set Output Schema, 'Use Output Header' must be set to 'True' \ + column names. Else, you do not need to add an Input Schema.", + "", + False, + False, + ) + + processor.addProperty( + "Use Output Header", + "Add True or False for whether you want to use an output \ + for your predictions.", + "False", + False, + False, + ) + + processor.addProperty( + "Output Schema", + "To set Output Schema, 'Use Output Header' must be set to 'True' \ If you want more descriptive column names for your predictions, then add an Output Schema. If there \ is more than one column name, write a comma separated list of column names. Else, H2O-3 will include \ - them by default", "", False, False) + them by default", + "", + False, + False, + ) def onSchedule(context): - """ onSchedule is where you load and read properties - this function is called 1 time when the processor is scheduled to run + """onSchedule is where you load and read properties + this function is called 1 time when the processor is scheduled to run """ global mojo_model h2o.init() @@ -101,37 +132,34 @@ def onSchedule(context): mojo_model = h2o.import_mojo(mojo_model_filepath) -class ContentExtract(object): - """ ContentExtract callback class is defined for reading streams of data through the session - and has a process function that accepts the input stream +class ContentExtract: + """ContentExtract callback class is defined for reading streams of data through the session + and has a process function that accepts the input stream """ + def __init__(self): self.content = None def process(self, input_stream): - """ Use codecs getReader to read that data - """ - self.content = codecs.getreader('utf-8')(input_stream).read() + """Use codecs getReader to read that data""" + self.content = codecs.getreader("utf-8")(input_stream).read() return len(self.content) -class ContentWrite(object): - """ ContentWrite callback class is defined for writing streams of data through the session - """ +class ContentWrite: + """ContentWrite callback class is defined for writing streams of data through the session""" + def __init__(self, data): self.content = data def process(self, output_stream): - """ Use codecs getWriter to write data encoded to the stream - """ - codecs.getwriter('utf-8')(output_stream).write(self.content) + """Use codecs getWriter to write data encoded to the stream""" + codecs.getwriter("utf-8")(output_stream).write(self.content) return len(self.content) def onTrigger(context, session): - """ onTrigger is executed and passed processor context and session - """ - global mojo_model + """onTrigger is executed and passed processor context and session""" flow_file = session.get() if flow_file is not None: # read test data of flow file content into read_cb.content @@ -167,5 +195,5 @@ def onTrigger(context, session): for i in range(len(pred_header)): ff_attr_name = pred_header[i] + "_pred_0" flow_file.addAttribute(ff_attr_name, str(preds_pd_df.at[0, pred_header[i]])) - log.info("getAttribute({}): {}".format(ff_attr_name, flow_file.getAttribute(ff_attr_name))) + log.info(f"getAttribute({ff_attr_name}): {flow_file.getAttribute(ff_attr_name)}") session.transfer(flow_file, REL_SUCCESS) diff --git a/extensions/python/pythonprocessors/nifi_python_processors/utils/dependency_installer.py b/extensions/python/pythonprocessors/nifi_python_processors/utils/dependency_installer.py index 0365f20052..8a7c0f3414 100644 --- a/extensions/python/pythonprocessors/nifi_python_processors/utils/dependency_installer.py +++ b/extensions/python/pythonprocessors/nifi_python_processors/utils/dependency_installer.py @@ -1,7 +1,7 @@ import ast -import sys -import subprocess import os +import subprocess +import sys # Extract the list of PIP dependency packages from the visited processor class AST node @@ -16,11 +16,11 @@ def visit_ClassDef(self, node): # Iterate through the body of the class to find the ProcessorDetails nested class for child in node.body: - if isinstance(child, ast.ClassDef) and child.name == 'ProcessorDetails': + if isinstance(child, ast.ClassDef) and child.name == "ProcessorDetails": # Iterate through the nodes of the 'ProcessorDetails' class for detail in child.body: # Check if the child node is an assignment of the 'dependencies' member variable - if isinstance(detail, ast.Assign) and detail.targets[0].id == 'dependencies': + if isinstance(detail, ast.Assign) and detail.targets[0].id == "dependencies": # Iterate through values of the 'dependencies' list member variable for elt in detail.value.elts: # Check if the element is a string constant and add it to the dependencies list @@ -31,8 +31,8 @@ def visit_ClassDef(self, node): def extract_dependencies(file_path): - class_name = file_path.split(os.sep)[-1].split('.')[0] - with open(file_path, 'r') as file: + class_name = file_path.split(os.sep)[-1].split(".")[0] + with open(file_path, "r") as file: code = file.read() tree = ast.parse(code) @@ -42,10 +42,10 @@ def extract_dependencies(file_path): def has_progress_bar() -> bool: - return sys.version_info.major > 3 or (sys.version_info.major == 3 and sys.version_info.minor >= 9) + return sys.version_info >= (3, 9) -if __name__ == '__main__': +if __name__ == "__main__": if len(sys.argv) < 2: sys.exit(1) @@ -53,20 +53,28 @@ def has_progress_bar() -> bool: # --no-cache-dir is used to be in line with NiFi's dependency install behavior if has_progress_bar(): - command = [sys.executable, "-m", "pip", "install", "--no-cache-dir", "--progress-bar", "off"] + command = [ + sys.executable, + "-m", + "pip", + "install", + "--no-cache-dir", + "--progress-bar", + "off", + ] else: command = [sys.executable, "-m", "pip", "install", "--no-cache-dir"] dependencies_found = False for i in range(1, len(sys.argv)): if "requirements.txt" in sys.argv[i]: command += ["-r", sys.argv[i]] - print("Installing dependencies from requirements file: {}".format(sys.argv[i])) + print(f"Installing dependencies from requirements file: {sys.argv[i]}") dependencies_found = True else: dependencies = extract_dependencies(sys.argv[i]) if dependencies: dependencies_found = True - print("Installing dependencies for processor {}: {}".format(sys.argv[i], str(dependencies))) + print(f"Installing dependencies for processor {sys.argv[i]}: {dependencies!s}") command += dependencies if dependencies_found: @@ -74,7 +82,7 @@ def has_progress_bar() -> bool: subprocess.check_call(command) print("Done installing dependencies for MiNiFi python processors.") except subprocess.CalledProcessError as e: - print("Error occurred while installing dependencies: {}".format(e)) + print(f"Error occurred while installing dependencies: {e}") sys.exit(1) else: print("No dependencies were found for MiNiFi python processors. No python packages were installed.") diff --git a/extensions/python/pythonprocessors/nifiapi/componentstate.py b/extensions/python/pythonprocessors/nifiapi/componentstate.py index f6cf7e6b70..51fcb0db32 100644 --- a/extensions/python/pythonprocessors/nifiapi/componentstate.py +++ b/extensions/python/pythonprocessors/nifiapi/componentstate.py @@ -14,7 +14,7 @@ # limitations under the License. from enum import Enum -from typing import Dict + from minifi_native import StateManager as CppFlowFile @@ -24,7 +24,7 @@ class Scope(Enum): class StateMap: - def __init__(self, state_map: Dict[str, str]): + def __init__(self, state_map: dict[str, str]): self.state_map = state_map if state_map is not None else {} def getStateVersion(self) -> int: @@ -36,7 +36,7 @@ def get(self, key) -> str: return self.state_map[key] - def toMap(self) -> Dict[str, str]: + def toMap(self) -> dict[str, str]: return self.state_map @@ -48,7 +48,7 @@ class StateManager: def __init__(self, cpp_state_manager: CppFlowFile): self.cpp_state_manager = cpp_state_manager - def setState(self, state: Dict[str, str], scope: Scope) -> bool: + def setState(self, state: dict[str, str], scope: Scope) -> bool: try: return self.cpp_state_manager.set(state) except Exception as exception: @@ -60,7 +60,7 @@ def getState(self, scope: Scope) -> StateMap: except Exception as exception: raise StateException("Get state failed") from exception - def replace(self, old_state: StateMap, new_values: Dict[str, str], scope: Scope) -> bool: + def replace(self, old_state: StateMap, new_values: dict[str, str], scope: Scope) -> bool: try: return self.cpp_state_manager.replace(old_state.toMap(), new_values) except Exception as exception: diff --git a/extensions/python/pythonprocessors/nifiapi/decorators.py b/extensions/python/pythonprocessors/nifiapi/decorators.py index 502817f621..a0aea48002 100644 --- a/extensions/python/pythonprocessors/nifiapi/decorators.py +++ b/extensions/python/pythonprocessors/nifiapi/decorators.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + def trigger_serially(cls): cls._trigger_serially = True return cls diff --git a/extensions/python/pythonprocessors/nifiapi/documentation.py b/extensions/python/pythonprocessors/nifiapi/documentation.py index e09f0bb7e8..746f8051a8 100644 --- a/extensions/python/pythonprocessors/nifiapi/documentation.py +++ b/extensions/python/pythonprocessors/nifiapi/documentation.py @@ -1,3 +1,5 @@ +from typing import ClassVar + # Licensed to the Apache Software Foundation (ASF) under one or more # contributor license agreements. See the NOTICE file distributed with # this work for additional information regarding copyright ownership. @@ -16,7 +18,7 @@ class ProcessorConfiguration: class Java: - implements = ['org.apache.nifi.python.processor.documentation.ProcessorConfigurationDetails'] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.documentation.ProcessorConfigurationDetails"] def __init__(self, processor_type: str, configuration: str): self.processor_type = processor_type @@ -29,23 +31,37 @@ def getConfiguration(self): return self.configuration -def use_case(description: str, configuration: str = None, notes: str = None, keywords: list[str] = None): +def use_case( + description: str, + configuration: str | None = None, + notes: str | None = None, + keywords: list[str] | None = None, +): """Decorator to explain how to perform a specific use case with a given processor""" + def decorator(func): return func + return decorator -def multi_processor_use_case(description: str, configurations: list[ProcessorConfiguration], notes: str = None, keywords: list[str] = None): +def multi_processor_use_case( + description: str, + configurations: list[ProcessorConfiguration], + notes: str | None = None, + keywords: list[str] | None = None, +): """Decorator to explain how to perform a specific use case that involves the decorated Processor, in addition to additional Processors""" + def decorator(func): return func + return decorator class UseCaseDetails: class Java: - implements = ['org.apache.nifi.python.processor.documentation.UseCaseDetails'] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.documentation.UseCaseDetails"] def __init__(self, description: str, notes: str, keywords: list[str], configuration: str): self.description = description @@ -71,9 +87,15 @@ def __str__(self): class MultiProcessorUseCaseDetails: class Java: - implements = ['org.apache.nifi.python.processor.documentation.MultiProcessorUseCaseDetails'] - - def __init__(self, description: str, notes: str, keywords: list[str], configurations: list[ProcessorConfiguration]): + implements: ClassVar[list] = ["org.apache.nifi.python.processor.documentation.MultiProcessorUseCaseDetails"] + + def __init__( + self, + description: str, + notes: str, + keywords: list[str], + configurations: list[ProcessorConfiguration], + ): self.description = description self.notes = notes self.keywords = keywords @@ -97,17 +119,19 @@ def __str__(self): class PropertyDescription: class Java: - implements = ['org.apache.nifi.python.processor.documentation.PropertyDescription'] - - def __init__(self, - name: str, - description: str, - display_name: str = None, - required: bool = False, - sensitive: bool = True, - default_value: str = None, - expression_language_scope: str = None, - controller_service_definition: str = None): + implements: ClassVar[list] = ["org.apache.nifi.python.processor.documentation.PropertyDescription"] + + def __init__( + self, + name: str, + description: str, + display_name: str | None = None, + required: bool = False, + sensitive: bool = True, + default_value: str | None = None, + expression_language_scope: str | None = None, + controller_service_definition: str | None = None, + ): self.name = name self.description = description self.display_name = display_name diff --git a/extensions/python/pythonprocessors/nifiapi/flowfilesource.py b/extensions/python/pythonprocessors/nifiapi/flowfilesource.py index 74041d7804..ee4ec925f7 100644 --- a/extensions/python/pythonprocessors/nifiapi/flowfilesource.py +++ b/extensions/python/pythonprocessors/nifiapi/flowfilesource.py @@ -14,10 +14,12 @@ # limitations under the License. import traceback -from .properties import ProcessContext as ProcessContextProxy from abc import abstractmethod -from minifi_native import ProcessContext, ProcessSession, FlowFile + +from minifi_native import FlowFile, ProcessContext, ProcessSession + from .processorbase import ProcessorBase, WriteCallback +from .properties import ProcessContext as ProcessContextProxy class FlowFileSourceResult: @@ -47,7 +49,7 @@ def onTrigger(self, context: ProcessContext, session: ProcessSession): if not result: return except Exception: - self.logger.error("Failed to create flow file due to error:\n{}".format(traceback.format_exc())) + self.logger.error(f"Failed to create flow file due to error:\n{traceback.format_exc()}") return flow_file = self.createFlowFile(session, result.getAttributes(), result.getContents()) diff --git a/extensions/python/pythonprocessors/nifiapi/flowfiletransform.py b/extensions/python/pythonprocessors/nifiapi/flowfiletransform.py index c72ebfb54c..5ec3854fe6 100644 --- a/extensions/python/pythonprocessors/nifiapi/flowfiletransform.py +++ b/extensions/python/pythonprocessors/nifiapi/flowfiletransform.py @@ -15,7 +15,9 @@ import traceback from abc import abstractmethod + from minifi_native import ProcessContext, ProcessSession + from .processorbase import ProcessorBase, WriteCallback from .properties import FlowFile as FlowFileProxy from .properties import ProcessContext as ProcessContextProxy @@ -53,14 +55,16 @@ def onTrigger(self, context: ProcessContext, session: ProcessSession): try: result = self.transform(context_proxy, flow_file_proxy) except Exception: - self.logger.error("Failed to transform flow file due to error:\n{}".format(traceback.format_exc())) + self.logger.error(f"Failed to transform flow file due to error:\n{traceback.format_exc()}") session.remove(flow_file) session.transfer(original_flow_file, self.REL_FAILURE) return if result.getRelationship() == "original": session.remove(flow_file) - self.logger.error("Result relationship cannot be 'original', it is reserved for the original flow file, and transferred automatically in non-failure cases.") + self.logger.error( + "Result relationship cannot be 'original', it is reserved for the original flow file, and transferred automatically in non-failure cases." + ) session.transfer(original_flow_file, self.REL_FAILURE) return @@ -71,7 +75,9 @@ def onTrigger(self, context: ProcessContext, session: ProcessSession): for name, value in result_attributes.items(): original_flow_file.setAttribute(name, value) if result.getContents() is not None: - self.logger.error("'failure' relationship should not have content, the original flow file will be transferred automatically in this case.") + self.logger.error( + "'failure' relationship should not have content, the original flow file will be transferred automatically in this case." + ) session.transfer(original_flow_file, self.REL_FAILURE) return diff --git a/extensions/python/pythonprocessors/nifiapi/processorbase.py b/extensions/python/pythonprocessors/nifiapi/processorbase.py index 2088b03ff9..5416ebc6f6 100644 --- a/extensions/python/pythonprocessors/nifiapi/processorbase.py +++ b/extensions/python/pythonprocessors/nifiapi/processorbase.py @@ -14,10 +14,16 @@ # limitations under the License. from abc import ABC, abstractmethod -from typing import List -from .properties import ExpressionLanguageScope, PropertyDescriptor, translateStandardValidatorToMiNiFiPropertype, MinifiPropertyTypes + +from minifi_native import OutputStream, ProcessContext, Processor, ProcessSession + +from .properties import ( + ExpressionLanguageScope, + MinifiPropertyTypes, + PropertyDescriptor, + translateStandardValidatorToMiNiFiPropertype, +) from .properties import ProcessContext as ProcessContextProxy -from minifi_native import OutputStream, Processor, ProcessContext, ProcessSession class WriteCallback: @@ -37,38 +43,53 @@ class ProcessorBase(ABC): REL_ORIGINAL = None def describe(self, processor: Processor): - if hasattr(self, 'ProcessorDetails') and hasattr(self.ProcessorDetails, 'description'): + if hasattr(self, "ProcessorDetails") and hasattr(self.ProcessorDetails, "description"): processor.setDescription(self.ProcessorDetails.description) else: processor.setDescription(self.__class__.__name__) - if hasattr(self, 'ProcessorDetails') and hasattr(self.ProcessorDetails, 'version'): + if hasattr(self, "ProcessorDetails") and hasattr(self.ProcessorDetails, "version"): processor.setVersion(self.ProcessorDetails.version) def onInitialize(self, processor: Processor): - get_dynamic_property_descriptor_attr = getattr(self, 'getDynamicPropertyDescriptor', None) + get_dynamic_property_descriptor_attr = getattr(self, "getDynamicPropertyDescriptor", None) if get_dynamic_property_descriptor_attr and callable(get_dynamic_property_descriptor_attr): processor.setSupportsDynamicProperties() self.supports_dynamic_properties = True else: self.supports_dynamic_properties = False - if hasattr(self, '_trigger_serially') and self._trigger_serially: + if hasattr(self, "_trigger_serially") and self._trigger_serially: processor.setSingleThreaded() for property in self.getPropertyDescriptors(): - expression_language_supported = True if property.expressionLanguageScope != ExpressionLanguageScope.NONE else False + expression_language_supported = property.expressionLanguageScope != ExpressionLanguageScope.NONE property_type_code = translateStandardValidatorToMiNiFiPropertype(property.validators) # MiNiFi C++ does not support validators for expression language enabled properties - if expression_language_supported and property_type_code is not None and property_type_code != MinifiPropertyTypes.NON_BLANK_TYPE: - self.logger.warn("Property '{}' has validators defined, but since it also supports Expression Language, the validators will be ignored.".format(property.name)) + if ( + expression_language_supported + and property_type_code is not None + and property_type_code != MinifiPropertyTypes.NON_BLANK_TYPE + ): + self.logger.warn( + f"Property '{property.name}' has validators defined, but since it also supports Expression Language, the validators will be ignored." + ) property_type_code = None # MiNiFi C++ does not support dependant properties, so if a property depends on another property, it should not be required - is_required = True if property.required and not property.dependencies else False - processor.addProperty(property.name, property.description, property.defaultValue, is_required, expression_language_supported, - property.sensitive, property_type_code, property.allowableValues, property.controllerServiceDefinition) + is_required = bool(property.required and not property.dependencies) + processor.addProperty( + property.name, + property.description, + property.defaultValue, + is_required, + expression_language_supported, + property.sensitive, + property_type_code, + property.allowableValues, + property.controllerServiceDefinition, + ) def onScheduled(self, context_proxy: ProcessContextProxy): pass @@ -81,5 +102,5 @@ def onSchedule(self, context: ProcessContext): def onTrigger(self, context: ProcessContext, session: ProcessSession): pass - def getPropertyDescriptors(self) -> List[PropertyDescriptor]: + def getPropertyDescriptors(self) -> list[PropertyDescriptor]: return [] diff --git a/extensions/python/pythonprocessors/nifiapi/properties.py b/extensions/python/pythonprocessors/nifiapi/properties.py index fe0dab092f..7e776a1839 100644 --- a/extensions/python/pythonprocessors/nifiapi/properties.py +++ b/extensions/python/pythonprocessors/nifiapi/properties.py @@ -14,10 +14,15 @@ # limitations under the License. from enum import Enum -from typing import List, Dict -from minifi_native import ProcessSession, timePeriodStringToMilliseconds, dataSizeStringToBytes + from minifi_native import FlowFile as CppFlowFile from minifi_native import ProcessContext as CppProcessContext +from minifi_native import ( + ProcessSession, + dataSizeStringToBytes, + timePeriodStringToMilliseconds, +) + from .componentstate import StateManager @@ -93,7 +98,7 @@ class MinifiPropertyTypes: NUMBER_TYPE = 7 -def translateStandardValidatorToMiNiFiPropertype(validators: List[int]) -> int: +def translateStandardValidatorToMiNiFiPropertype(validators: list[int]) -> int: if validators is None or len(validators) == 0 or len(validators) > 1: return None @@ -127,7 +132,14 @@ def __init__(self, property_descriptor, *dependent_values): class ResourceDefinition: - def __init__(self, allow_multiple=False, allow_file=True, allow_url=False, allow_directory=False, allow_text=False): + def __init__( + self, + allow_multiple=False, + allow_file=True, + allow_url=False, + allow_directory=False, + allow_text=False, + ): self.allow_multiple = allow_multiple self.allow_file = allow_file self.allow_url = allow_url @@ -142,10 +154,22 @@ class ExpressionLanguageScope(Enum): class PropertyDescriptor: - def __init__(self, name: str, description: str, required: bool = False, sensitive: bool = False, - display_name: str = None, default_value: str = None, allowable_values: List[str] = None, - dependencies: List[PropertyDependency] = None, expression_language_scope: ExpressionLanguageScope = ExpressionLanguageScope.NONE, - dynamic: bool = False, validators: List[int] = None, resource_definition: ResourceDefinition = None, controller_service_definition: str = None): + def __init__( + self, + name: str, + description: str, + required: bool = False, + sensitive: bool = False, + display_name: str | None = None, + default_value: str | None = None, + allowable_values: list[str] | None = None, + dependencies: list[PropertyDependency] | None = None, + expression_language_scope: ExpressionLanguageScope = ExpressionLanguageScope.NONE, + dynamic: bool = False, + validators: list[int] | None = None, + resource_definition: ResourceDefinition = None, + controller_service_definition: str | None = None, + ): if validators is None: validators = [StandardValidators.ALWAYS_VALID] @@ -165,20 +189,20 @@ def __init__(self, name: str, description: str, required: bool = False, sensitiv class TimeUnit(Enum): - NANOSECONDS = "NANOSECONDS", - MICROSECONDS = "MICROSECONDS", - MILLISECONDS = "MILLISECONDS", - SECONDS = "SECONDS", - MINUTES = "MINUTES", - HOURS = "HOURS", + NANOSECONDS = ("NANOSECONDS",) + MICROSECONDS = ("MICROSECONDS",) + MILLISECONDS = ("MILLISECONDS",) + SECONDS = ("SECONDS",) + MINUTES = ("MINUTES",) + HOURS = ("HOURS",) DAYS = "DAYS" class DataUnit(Enum): - B = "B", - KB = "KB", - MB = "MB", - GB = "GB", + B = ("B",) + KB = ("KB",) + MB = ("MB",) + GB = ("GB",) TB = "TB" @@ -201,7 +225,15 @@ def getAttributes(self): class PythonPropertyValue: - def __init__(self, cpp_context: CppProcessContext, name: str, string_value: str, el_supported: bool, controller_service_definition: str, is_dynamic: bool = False): + def __init__( + self, + cpp_context: CppProcessContext, + name: str, + string_value: str, + el_supported: bool, + controller_service_definition: str, + is_dynamic: bool = False, + ): self.cpp_context = cpp_context self.value = string_value self.name = name @@ -223,7 +255,7 @@ def asInteger(self) -> int: def asBoolean(self) -> bool: if not self.value: return None - return self.value.lower() == 'true' + return self.value.lower() == "true" def asFloat(self) -> float: if not self.value: @@ -241,13 +273,13 @@ def asTimePeriod(self, time_unit: TimeUnit) -> int: if time_unit == TimeUnit.MILLISECONDS: return milliseconds if time_unit == TimeUnit.SECONDS: - return int(round(milliseconds / 1000)) + return round(milliseconds / 1000) if time_unit == TimeUnit.MINUTES: - return int(round(milliseconds / 1000 / 60)) + return round(milliseconds / 1000 / 60) if time_unit == TimeUnit.HOURS: - return int(round(milliseconds / 1000 / 60 / 60)) + return round(milliseconds / 1000 / 60 / 60) if time_unit == TimeUnit.DAYS: - return int(round(milliseconds / 1000 / 60 / 60 / 24)) + return round(milliseconds / 1000 / 60 / 60 / 24) return 0 def asDataSize(self, data_unit: DataUnit) -> float: @@ -275,11 +307,20 @@ def evaluateAttributeExpressions(self, flow_file: FlowFile = None): getter = self.cpp_context.getDynamicProperty if self.is_dynamic else self.cpp_context.getProperty args = () if flow_file is None else (flow_file.cpp_flow_file,) new_string_value = getter(self.name, *args) - return PythonPropertyValue(self.cpp_context, self.name, new_string_value, self.el_supported, self.controller_service_definition, self.is_dynamic) + return PythonPropertyValue( + self.cpp_context, + self.name, + new_string_value, + self.el_supported, + self.controller_service_definition, + self.is_dynamic, + ) def asControllerService(self): if not self.controller_service_definition: - raise Exception("Controller Service definition is not set, getProperty must be called with a property descriptor instead of string value") + raise RuntimeError( + "Controller Service definition is not set, getProperty must be called with a property descriptor instead of string value" + ) return self.cpp_context.getControllerService(self.value, self.controller_service_definition) @@ -305,7 +346,14 @@ def getProperty(self, descriptor) -> PythonPropertyValue: property_value = self.cpp_context.getRawDynamicProperty(property_name) if property_value is not None: is_dynamic = True - return PythonPropertyValue(self.cpp_context, property_name, property_value, expression_language_support, controller_service_definition, is_dynamic) + return PythonPropertyValue( + self.cpp_context, + property_name, + property_value, + expression_language_support, + controller_service_definition, + is_dynamic, + ) def getStateManager(self) -> StateManager: return StateManager(self.cpp_context.getStateManager()) @@ -313,8 +361,8 @@ def getStateManager(self) -> StateManager: def getName(self) -> str: return self.cpp_context.getName() - def getProperties(self) -> Dict[PropertyDescriptor, str]: - properties = dict() + def getProperties(self) -> dict[PropertyDescriptor, str]: + properties = {} cpp_properties = self.cpp_context.getProperties() for property_descriptor in self.processor.getPropertyDescriptors(): diff --git a/extensions/python/pythonprocessors/nifiapi/recordtransform.py b/extensions/python/pythonprocessors/nifiapi/recordtransform.py index bdeb2ffb60..c3aaff4140 100644 --- a/extensions/python/pythonprocessors/nifiapi/recordtransform.py +++ b/extensions/python/pythonprocessors/nifiapi/recordtransform.py @@ -13,10 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -import traceback import json +import traceback from abc import abstractmethod -from minifi_native import ProcessContext, ProcessSession, Processor + +from minifi_native import ProcessContext, Processor, ProcessSession + from .processorbase import ProcessorBase from .properties import FlowFile as FlowFileProxy from .properties import ProcessContext as ProcessContextProxy @@ -63,24 +65,44 @@ def getPartition(self): class RecordTransform(ProcessorBase): RECORD_READER = PropertyDescriptor( - name='Record Reader', - display_name='Record Reader', - description='''Specifies the Controller Service to use for reading incoming data''', + name="Record Reader", + display_name="Record Reader", + description="""Specifies the Controller Service to use for reading incoming data""", required=True, - controller_service_definition='RecordSetReader' + controller_service_definition="RecordSetReader", ) RECORD_WRITER = PropertyDescriptor( - name='Record Writer', - display_name='Record Writer', - description='''Specifies the Controller Service to use for writing out the records''', + name="Record Writer", + display_name="Record Writer", + description="""Specifies the Controller Service to use for writing out the records""", required=True, - controller_service_definition='RecordSetWriter', + controller_service_definition="RecordSetWriter", ) def onInitialize(self, processor: Processor): - super(RecordTransform, self).onInitialize(processor) - processor.addProperty(self.RECORD_READER.name, self.RECORD_READER.description, None, self.RECORD_READER.required, False, False, None, None, self.RECORD_READER.controllerServiceDefinition) - processor.addProperty(self.RECORD_WRITER.name, self.RECORD_WRITER.description, None, self.RECORD_WRITER.required, False, False, None, None, self.RECORD_WRITER.controllerServiceDefinition) + super().onInitialize(processor) + processor.addProperty( + self.RECORD_READER.name, + self.RECORD_READER.description, + None, + self.RECORD_READER.required, + False, + False, + None, + None, + self.RECORD_READER.controllerServiceDefinition, + ) + processor.addProperty( + self.RECORD_WRITER.name, + self.RECORD_WRITER.description, + None, + self.RECORD_WRITER.required, + False, + False, + None, + None, + self.RECORD_WRITER.controllerServiceDefinition, + ) def onTrigger(self, context: ProcessContext, session: ProcessSession): flow_file = session.get() @@ -106,7 +128,7 @@ def onTrigger(self, context: ProcessContext, session: ProcessSession): session.transfer(flow_file, self.REL_FAILURE) return except Exception: - self.logger.error("Failed to read flow file records due to the following error:\n{}".format(traceback.format_exc())) + self.logger.error(f"Failed to read flow file records due to the following error:\n{traceback.format_exc()}") session.transfer(flow_file, self.REL_FAILURE) return @@ -120,7 +142,7 @@ def onTrigger(self, context: ProcessContext, session: ProcessSession): resultjson = None if result_record is None else json.dumps(result_record) results.append(__RecordTransformResult__(result, resultjson)) except Exception: - self.logger.error("Failed to transform record due to the following error:\n{}".format(traceback.format_exc())) + self.logger.error(f"Failed to transform record due to the following error:\n{traceback.format_exc()}") session.transfer(flow_file, self.REL_FAILURE) return @@ -139,7 +161,11 @@ def onTrigger(self, context: ProcessContext, session: ProcessSession): for single_partition_results in partitioned_results_list: partitioned_flow_file = session.create(flow_file) - record_writer.write([result.getRecordJson() for result in single_partition_results], partitioned_flow_file, session) + record_writer.write( + [result.getRecordJson() for result in single_partition_results], + partitioned_flow_file, + session, + ) if result.getRelationship() == "success": session.transfer(partitioned_flow_file, self.REL_SUCCESS) else: @@ -148,5 +174,7 @@ def onTrigger(self, context: ProcessContext, session: ProcessSession): session.transfer(flow_file, self.REL_ORIGINAL) @abstractmethod - def transform(self, context: ProcessContextProxy, record_json, schema, flowFile: FlowFileProxy) -> RecordTransformResult: + def transform( + self, context: ProcessContextProxy, record_json, schema, flowFile: FlowFileProxy + ) -> RecordTransformResult: pass diff --git a/extensions/python/pythonprocessors/nifiapi/relationship.py b/extensions/python/pythonprocessors/nifiapi/relationship.py index 2504c5d803..58fda3d907 100644 --- a/extensions/python/pythonprocessors/nifiapi/relationship.py +++ b/extensions/python/pythonprocessors/nifiapi/relationship.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + class Relationship: def __init__(self, name, description, auto_terminated=False): self.name = name diff --git a/extensions/python/tests/features/environment.py b/extensions/python/tests/features/environment.py index 648377cda3..77e1d908ea 100644 --- a/extensions/python/tests/features/environment.py +++ b/extensions/python/tests/features/environment.py @@ -12,63 +12,63 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import docker import os from pathlib import Path + from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def get_minifi_container_image(): - if 'MINIFI_TAG_PREFIX' in os.environ and 'MINIFI_VERSION' in os.environ: - minifi_tag_prefix = os.environ['MINIFI_TAG_PREFIX'] - minifi_version = os.environ['MINIFI_VERSION'] - return 'apacheminificpp:' + minifi_tag_prefix + minifi_version + if "MINIFI_TAG_PREFIX" in os.environ and "MINIFI_VERSION" in os.environ: + minifi_tag_prefix = os.environ["MINIFI_TAG_PREFIX"] + minifi_version = os.environ["MINIFI_VERSION"] + return "apacheminificpp:" + minifi_tag_prefix + minifi_version return "apacheminificpp:behave" def before_all(context): context.minifi_container_image = get_minifi_container_image() client: docker.DockerClient = docker.from_env() - is_fhs = 'MINIFI_INSTALLATION_TYPE=FHS' in str(client.images.get(context.minifi_container_image).history()) + is_fhs = "MINIFI_INSTALLATION_TYPE=FHS" in str(client.images.get(context.minifi_container_image).history()) pip3_install_command = "" - minifi_tag_prefix = os.environ['MINIFI_TAG_PREFIX'] if 'MINIFI_TAG_PREFIX' in os.environ else "" + minifi_tag_prefix = os.environ.get("MINIFI_TAG_PREFIX", "") if not minifi_tag_prefix: pip3_install_command = "RUN apk --update --no-cache add py3-pip" - minifi_python_dir_path = '/var/lib/nifi-minifi-cpp/minifi-python' if is_fhs else '/opt/minifi/minifi-current/minifi-python' - minifi_python_venv_parent = '/var/lib/nifi-minifi-cpp' if is_fhs else '/opt/minifi/minifi-current' - dockerfile = """ -FROM {base_image} + minifi_python_dir_path = ( + "/var/lib/nifi-minifi-cpp/minifi-python" if is_fhs else "/opt/minifi/minifi-current/minifi-python" + ) + minifi_python_venv_parent = "/var/lib/nifi-minifi-cpp" if is_fhs else "/opt/minifi/minifi-current" + dockerfile = f""" +FROM {context.minifi_container_image} USER root {pip3_install_command} USER minificpp -COPY RotatingForwarder.py {minifi_python_dir}/nifi_python_processors/RotatingForwarder.py -COPY SpecialPropertyTypeChecker.py {minifi_python_dir}/nifi_python_processors/SpecialPropertyTypeChecker.py -COPY ProcessContextInterfaceChecker.py {minifi_python_dir}/nifi_python_processors/ProcessContextInterfaceChecker.py -COPY CreateFlowFile.py {minifi_python_dir}/nifi_python_processors/CreateFlowFile.py -COPY FailureWithAttributes.py {minifi_python_dir}/nifi_python_processors/FailureWithAttributes.py -COPY subtractutils.py {minifi_python_dir}/nifi_python_processors/compute/subtractutils.py -COPY RelativeImporterProcessor.py {minifi_python_dir}/nifi_python_processors/compute/processors/RelativeImporterProcessor.py -COPY multiplierutils.py {minifi_python_dir}/nifi_python_processors/compute/processors/multiplierutils.py -COPY CreateNothing.py {minifi_python_dir}/nifi_python_processors/CreateNothing.py -COPY FailureWithContent.py {minifi_python_dir}/nifi_python_processors/FailureWithContent.py -COPY TransferToOriginal.py {minifi_python_dir}/nifi_python_processors/TransferToOriginal.py -COPY SetRecordField.py {minifi_python_dir}/nifi_python_processors/SetRecordField.py -COPY TestStateManager.py {minifi_python_dir}/nifi_python_processors/TestStateManager.py -COPY NifiStyleLogDynamicProperties.py {minifi_python_dir}/nifi_python_processors/NifiStyleLogDynamicProperties.py -COPY LogDynamicProperties.py {minifi_python_dir}/LogDynamicProperties.py -COPY ExpressionLanguagePropertyWithValidator.py {minifi_python_dir}/nifi_python_processors/ExpressionLanguagePropertyWithValidator.py -COPY EvaluateExpressionLanguageChecker.py {minifi_python_dir}/nifi_python_processors/EvaluateExpressionLanguageChecker.py -COPY SleepForever.py {minifi_python_dir}/nifi_python_processors/SleepForever.py -COPY SingleThreadedSleepForever.py {minifi_python_dir}/nifi_python_processors/SingleThreadedSleepForever.py -COPY MinifiSleepForever.py {minifi_python_dir}/MinifiSleepForever.py -COPY SingleThreadedMinifiSleepForever.py {minifi_python_dir}/SingleThreadedMinifiSleepForever.py +COPY RotatingForwarder.py {minifi_python_dir_path}/nifi_python_processors/RotatingForwarder.py +COPY SpecialPropertyTypeChecker.py {minifi_python_dir_path}/nifi_python_processors/SpecialPropertyTypeChecker.py +COPY ProcessContextInterfaceChecker.py {minifi_python_dir_path}/nifi_python_processors/ProcessContextInterfaceChecker.py +COPY CreateFlowFile.py {minifi_python_dir_path}/nifi_python_processors/CreateFlowFile.py +COPY FailureWithAttributes.py {minifi_python_dir_path}/nifi_python_processors/FailureWithAttributes.py +COPY subtractutils.py {minifi_python_dir_path}/nifi_python_processors/compute/subtractutils.py +COPY RelativeImporterProcessor.py {minifi_python_dir_path}/nifi_python_processors/compute/processors/RelativeImporterProcessor.py +COPY multiplierutils.py {minifi_python_dir_path}/nifi_python_processors/compute/processors/multiplierutils.py +COPY CreateNothing.py {minifi_python_dir_path}/nifi_python_processors/CreateNothing.py +COPY FailureWithContent.py {minifi_python_dir_path}/nifi_python_processors/FailureWithContent.py +COPY TransferToOriginal.py {minifi_python_dir_path}/nifi_python_processors/TransferToOriginal.py +COPY SetRecordField.py {minifi_python_dir_path}/nifi_python_processors/SetRecordField.py +COPY TestStateManager.py {minifi_python_dir_path}/nifi_python_processors/TestStateManager.py +COPY NifiStyleLogDynamicProperties.py {minifi_python_dir_path}/nifi_python_processors/NifiStyleLogDynamicProperties.py +COPY LogDynamicProperties.py {minifi_python_dir_path}/LogDynamicProperties.py +COPY ExpressionLanguagePropertyWithValidator.py {minifi_python_dir_path}/nifi_python_processors/ExpressionLanguagePropertyWithValidator.py +COPY EvaluateExpressionLanguageChecker.py {minifi_python_dir_path}/nifi_python_processors/EvaluateExpressionLanguageChecker.py +COPY SleepForever.py {minifi_python_dir_path}/nifi_python_processors/SleepForever.py +COPY SingleThreadedSleepForever.py {minifi_python_dir_path}/nifi_python_processors/SingleThreadedSleepForever.py +COPY MinifiSleepForever.py {minifi_python_dir_path}/MinifiSleepForever.py +COPY SingleThreadedMinifiSleepForever.py {minifi_python_dir_path}/SingleThreadedMinifiSleepForever.py RUN python3 -m venv {minifi_python_venv_parent}/venv - """.format(base_image=context.minifi_container_image, - pip3_install_command=pip3_install_command, - minifi_python_dir=minifi_python_dir_path, - minifi_python_venv_parent=minifi_python_venv_parent) + """ build_context_path = str(Path(__file__).resolve().parent / "resources") files_on_context = {} @@ -80,7 +80,7 @@ def before_all(context): builder = DockerImageBuilder( image_tag="apacheminificpp-python:latest", dockerfile_content=dockerfile, - files_on_context=files_on_context + files_on_context=files_on_context, ) builder.build() @@ -89,7 +89,7 @@ def is_conda_available_in_minifi_image(context): client: docker.DockerClient = docker.from_env() container = client.containers.create( image=context.minifi_container_image, - command=['conda', '--version'], + command=["conda", "--version"], ) try: container.start() @@ -99,26 +99,29 @@ def is_conda_available_in_minifi_image(context): container.remove(force=True) return False - return result.decode('utf-8').startswith('conda ') + return result.decode("utf-8").startswith("conda ") def get_minifi_image_python_version(context): client: docker.DockerClient = docker.from_env() result = client.containers.run( image=context.minifi_container_image, - command=['python3', '-c', 'import platform; print(platform.python_version())'], - remove=True + command=["python3", "-c", "import platform; print(platform.python_version())"], + remove=True, ) - python_ver_str = result.decode('utf-8') - return tuple(map(int, python_ver_str.split('.'))) + python_ver_str = result.decode("utf-8") + return tuple(map(int, python_ver_str.split("."))) def before_scenario(context, scenario): - if "USE_NIFI_PYTHON_PROCESSORS_WITH_LANGCHAIN" in scenario.effective_tags: - if not is_conda_available_in_minifi_image(context) and get_minifi_image_python_version(context) < (3, 8, 1): - scenario.skip("NiFi Python processor tests use langchain library which requires Python 3.8.1 or later.") - return + if ( + "USE_NIFI_PYTHON_PROCESSORS_WITH_LANGCHAIN" in scenario.effective_tags + and not is_conda_available_in_minifi_image(context) + and get_minifi_image_python_version(context) < (3, 8, 1) + ): + scenario.skip("NiFi Python processor tests use langchain library which requires Python 3.8.1 or later.") + return common_before_scenario(context, scenario) context.minifi_container_image = "apacheminificpp-python:latest" diff --git a/extensions/python/tests/features/resources/CreateFlowFile.py b/extensions/python/tests/features/resources/CreateFlowFile.py index 948734982d..57267a465c 100644 --- a/extensions/python/tests/features/resources/CreateFlowFile.py +++ b/extensions/python/tests/features/resources/CreateFlowFile.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.flowfilesource import FlowFileSource, FlowFileSourceResult from nifiapi.properties import PropertyDescriptor from nifiapi.relationship import Relationship @@ -20,23 +22,23 @@ class CreateFlowFile(FlowFileSource): class Java: - implements = ['org.apache.nifi.python.processor.FlowFileSource'] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.FlowFileSource"] class ProcessorDetails: - version = '2.0.0-snapshot' - description = '''Test Python source processor.''' - tags = ['text', 'test', 'python', 'source'] + version = "2.0.0-snapshot" + description = """Test Python source processor.""" + tags: ClassVar[list] = ["text", "test", "python", "source"] FF_CONTENTS = PropertyDescriptor( - name='FlowFile Contents', - description='''The contents of the FlowFile.''', + name="FlowFile Contents", + description="""The contents of the FlowFile.""", required=True, - default_value='Hello World!' + default_value="Hello World!", ) - property_descriptors = [FF_CONTENTS] + property_descriptors: ClassVar[list] = [FF_CONTENTS] - REL_MULTILINE = Relationship(name='space', description='FlowFiles that contain space characters.') + REL_MULTILINE = Relationship(name="space", description="FlowFiles that contain space characters.") def __init__(self, **kwargs): pass @@ -52,7 +54,11 @@ def create(self, context): if contents is not None and isinstance(contents, str): contents_str = str.encode(contents) - if b' ' in contents_str: - return FlowFileSourceResult(relationship='space', attributes={"type": "space"}, contents=contents_str) + if b" " in contents_str: + return FlowFileSourceResult( + relationship="space", + attributes={"type": "space"}, + contents=contents_str, + ) - return FlowFileSourceResult(relationship='success', attributes={"type": "non-space"}, contents=contents) + return FlowFileSourceResult(relationship="success", attributes={"type": "non-space"}, contents=contents) diff --git a/extensions/python/tests/features/resources/CreateNothing.py b/extensions/python/tests/features/resources/CreateNothing.py index acef36dd85..7c6a8f69e7 100644 --- a/extensions/python/tests/features/resources/CreateNothing.py +++ b/extensions/python/tests/features/resources/CreateNothing.py @@ -12,22 +12,23 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.flowfilesource import FlowFileSource class CreateNothing(FlowFileSource): class Java: - implements = ['org.apache.nifi.python.processor.FlowFileSource'] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.FlowFileSource"] class ProcessorDetails: - version = '0.0.1-SNAPSHOT' - description = '''A Python processor for testing a use-case where the Source processor - does not create any output.''' - tags = ['test', 'python', 'source'] + version = "0.0.1-SNAPSHOT" + description = """A Python processor for testing a use-case where the Source processor + does not create any output.""" + tags: ClassVar[list] = ["test", "python", "source"] def __init__(self, **kwargs): pass def create(self, context): context.yield_resources() - return None diff --git a/extensions/python/tests/features/resources/EvaluateExpressionLanguageChecker.py b/extensions/python/tests/features/resources/EvaluateExpressionLanguageChecker.py index 269e7ca47c..344e4bab21 100644 --- a/extensions/python/tests/features/resources/EvaluateExpressionLanguageChecker.py +++ b/extensions/python/tests/features/resources/EvaluateExpressionLanguageChecker.py @@ -13,13 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult -from nifiapi.properties import PropertyDescriptor, ExpressionLanguageScope +from nifiapi.properties import ExpressionLanguageScope, PropertyDescriptor class EvaluateExpressionLanguageChecker(FlowFileTransform): class Java: - implements = ["org.apache.nifi.python.processor.FlowFileTransform"] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.FlowFileTransform"] class ProcessorDetails: version = "0.1.0" @@ -28,7 +30,7 @@ class ProcessorDetails: EL_PROPERTY = PropertyDescriptor( name="EL Property", description="EL property", - expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES + expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES, ) NON_EL_PROPERTY = PropertyDescriptor( @@ -36,10 +38,7 @@ class ProcessorDetails: description="Non EL property", ) - property_descriptors = [ - EL_PROPERTY, - NON_EL_PROPERTY - ] + property_descriptors: ClassVar[list] = [EL_PROPERTY, NON_EL_PROPERTY] def __init__(self, **kwargs): pass @@ -48,9 +47,7 @@ def getPropertyDescriptors(self): return self.property_descriptors def getDynamicPropertyDescriptor(self, propertyname): - return PropertyDescriptor(name=propertyname, - description="A user-defined property", - dynamic=True) + return PropertyDescriptor(name=propertyname, description="A user-defined property", dynamic=True) def transform(self, context, flowFile): el_property = context.getProperty(self.EL_PROPERTY) @@ -67,7 +64,9 @@ def transform(self, context, flowFile): non_el_property_value_evaluated = non_el_property.evaluateAttributeExpressions(flowFile).getValue() self.logger.info("Evaluated Non EL Property value: " + str(non_el_property_value_evaluated)) - non_existent_value = context.getProperty("non-existent-property").evaluateAttributeExpressions(flowFile).getValue() + non_existent_value = ( + context.getProperty("non-existent-property").evaluateAttributeExpressions(flowFile).getValue() + ) if non_existent_value is None: self.logger.info("Non-existent property value is empty") diff --git a/extensions/python/tests/features/resources/ExpressionLanguagePropertyWithValidator.py b/extensions/python/tests/features/resources/ExpressionLanguagePropertyWithValidator.py index 346f1ede41..28b889fda5 100644 --- a/extensions/python/tests/features/resources/ExpressionLanguagePropertyWithValidator.py +++ b/extensions/python/tests/features/resources/ExpressionLanguagePropertyWithValidator.py @@ -13,26 +13,31 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult -from nifiapi.properties import ExpressionLanguageScope, PropertyDescriptor, StandardValidators +from nifiapi.properties import ( + ExpressionLanguageScope, + PropertyDescriptor, + StandardValidators, +) class ExpressionLanguagePropertyWithValidator(FlowFileTransform): - class Java: - implements = ['org.apache.nifi.python.processor.FlowFileTransform'] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.FlowFileTransform"] class ProcessorDetails: - version = '1.2.3' + version = "1.2.3" description = "Test processor" - dependencies = [] + dependencies: ClassVar[list] = [] INTEGER_PROPERTY = PropertyDescriptor( name="Integer Property", description="A dummy integer property", required=True, validators=[StandardValidators.INTEGER_VALIDATOR], - expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES + expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES, ) def __init__(self, **kwargs): @@ -42,12 +47,10 @@ def getPropertyDescriptors(self): return [self.INTEGER_PROPERTY] def getDynamicPropertyDescriptor(self, propertyname): - return PropertyDescriptor(name=propertyname, - description="A user-defined property", - dynamic=True) + return PropertyDescriptor(name=propertyname, description="A user-defined property", dynamic=True) def transform(self, context, flow_file): integer_value = context.getProperty(self.INTEGER_PROPERTY).asInteger() - self.logger.info("Integer Property value: {}".format(integer_value)) + self.logger.info(f"Integer Property value: {integer_value}") - return FlowFileTransformResult('success', contents="content") + return FlowFileTransformResult("success", contents="content") diff --git a/extensions/python/tests/features/resources/LogDynamicProperties.py b/extensions/python/tests/features/resources/LogDynamicProperties.py index 727e383634..edd850d926 100644 --- a/extensions/python/tests/features/resources/LogDynamicProperties.py +++ b/extensions/python/tests/features/resources/LogDynamicProperties.py @@ -13,6 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. + def describe(processor): processor.setDescription("Test description.") @@ -25,11 +26,11 @@ def onInitialize(processor): def onTrigger(context, session): flow_file = session.get() property_value = context.getProperty("Static Property") - log.info("Static Property value: {}".format(property_value)) + log.info(f"Static Property value: {property_value}") dyn_property_value = context.getDynamicProperty("Dynamic Property") - log.info("Dynamic Property value: {}".format(dyn_property_value)) + log.info(f"Dynamic Property value: {dyn_property_value}") keys = context.getDynamicPropertyKeys() - log.info("dynamic property key count: {}".format(len(keys))) + log.info(f"dynamic property key count: {len(keys)}") for dynamic_property_key in keys: - log.info("dynamic property key: {}".format(dynamic_property_key)) + log.info(f"dynamic property key: {dynamic_property_key}") session.transfer(flow_file, REL_SUCCESS) diff --git a/extensions/python/tests/features/resources/NifiStyleLogDynamicProperties.py b/extensions/python/tests/features/resources/NifiStyleLogDynamicProperties.py index dc989f6b0a..adb1c4b523 100644 --- a/extensions/python/tests/features/resources/NifiStyleLogDynamicProperties.py +++ b/extensions/python/tests/features/resources/NifiStyleLogDynamicProperties.py @@ -13,26 +13,31 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult -from nifiapi.properties import ExpressionLanguageScope, PropertyDescriptor, StandardValidators +from nifiapi.properties import ( + ExpressionLanguageScope, + PropertyDescriptor, + StandardValidators, +) class NifiStyleLogDynamicProperties(FlowFileTransform): - class Java: - implements = ['org.apache.nifi.python.processor.FlowFileTransform'] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.FlowFileTransform"] class ProcessorDetails: - version = '1.2.3' + version = "1.2.3" description = "Test processor" - dependencies = [] + dependencies: ClassVar[list] = [] STATIC_PROPERTY = PropertyDescriptor( name="Static Property", description="A dummy static property", required=True, validators=[StandardValidators.NON_EMPTY_VALIDATOR], - expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES + expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES, ) def __init__(self, **kwargs): @@ -42,14 +47,12 @@ def getPropertyDescriptors(self): return [self.STATIC_PROPERTY] def getDynamicPropertyDescriptor(self, propertyname): - return PropertyDescriptor(name=propertyname, - description="A user-defined property", - dynamic=True) + return PropertyDescriptor(name=propertyname, description="A user-defined property", dynamic=True) def transform(self, context, flow_file): property_value = context.getProperty("Static Property") - self.logger.info("Static Property value: {}".format(property_value.getValue())) + self.logger.info(f"Static Property value: {property_value.getValue()}") dyn_property_value = context.getProperty("Dynamic Property") - self.logger.info("Dynamic Property value: {}".format(dyn_property_value.getValue())) + self.logger.info(f"Dynamic Property value: {dyn_property_value.getValue()}") - return FlowFileTransformResult('success', contents="content") + return FlowFileTransformResult("success", contents="content") diff --git a/extensions/python/tests/features/resources/ProcessContextInterfaceChecker.py b/extensions/python/tests/features/resources/ProcessContextInterfaceChecker.py index 5d0136adef..b086d6e679 100644 --- a/extensions/python/tests/features/resources/ProcessContextInterfaceChecker.py +++ b/extensions/python/tests/features/resources/ProcessContextInterfaceChecker.py @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult from nifiapi.properties import PropertyDescriptor, StandardValidators from nifiapi.relationship import Relationship @@ -22,33 +24,34 @@ class ProcessContextInterfaceChecker(FlowFileTransform): """ Checks ProcessContext interface """ + SECRET_PASSWORD = PropertyDescriptor( name="Secret Password", description="Password to access", validators=[StandardValidators.NON_EMPTY_VALIDATOR], default_value="mysecret", required=True, - sensitive=True + sensitive=True, ) REQUEST_TIMEOUT = PropertyDescriptor( name="Request Timeout", description="Timeout for the request", validators=[StandardValidators.TIME_PERIOD_VALIDATOR], default_value="60 sec", - required=True + required=True, ) WISH_COUNT = PropertyDescriptor( name="Wish Count", description="Number of wishes", default_value="3", validators=[StandardValidators.POSITIVE_INTEGER_VALIDATOR], - required=True + required=True, ) - property_descriptors = [ + property_descriptors: ClassVar[list] = [ SECRET_PASSWORD, REQUEST_TIMEOUT, - WISH_COUNT + WISH_COUNT, ] def __init__(self, **kwargs): @@ -66,15 +69,20 @@ def transform(self, context, flowFile): return FlowFileTransformResult("failure") property_names = [property.name for property in properties] - if "Secret Password" not in property_names or "Request Timeout" not in property_names or "Wish Count" not in property_names: + if ( + "Secret Password" not in property_names + or "Request Timeout" not in property_names + or "Wish Count" not in property_names + ): return FlowFileTransformResult("failure") for property in properties: - if property.name == "Secret Password" and properties[property] != "mysecret": - return FlowFileTransformResult("failure") - elif property.name == "Request Timeout" and properties[property] != "60 sec": - return FlowFileTransformResult("failure") - elif property.name == "Wish Count" and properties[property] != "3": + if ( + (property.name == "Secret Password" and properties[property] != "mysecret") + or (property.name == "Request Timeout" and properties[property] != "60 sec") + or property.name == "Wish Count" + and properties[property] != "3" + ): return FlowFileTransformResult("failure") secret_password = context.getProperty(self.SECRET_PASSWORD).getValue() diff --git a/extensions/python/tests/features/resources/RelativeImporterProcessor.py b/extensions/python/tests/features/resources/RelativeImporterProcessor.py index 1ec2d254f5..1f2a0e4135 100644 --- a/extensions/python/tests/features/resources/RelativeImporterProcessor.py +++ b/extensions/python/tests/features/resources/RelativeImporterProcessor.py @@ -14,8 +14,9 @@ # limitations under the License. from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult -from .multiplierutils import double + from ..subtractutils import minus_ten +from .multiplierutils import double class RelativeImporterProcessor(FlowFileTransform): @@ -26,4 +27,4 @@ def transform(self, context, flowFile): number = 1000 number = double(number) number = minus_ten(number) - return FlowFileTransformResult("success", contents="The final result is {}".format(number)) + return FlowFileTransformResult("success", contents=f"The final result is {number}") diff --git a/extensions/python/tests/features/resources/RotatingForwarder.py b/extensions/python/tests/features/resources/RotatingForwarder.py index 09df09fbf5..9fa1290c12 100644 --- a/extensions/python/tests/features/resources/RotatingForwarder.py +++ b/extensions/python/tests/features/resources/RotatingForwarder.py @@ -20,6 +20,7 @@ class RotatingForwarder(FlowFileTransform): """ Forwards flow files to a different relationship each time it is called in a round robin manner. """ + def __init__(self, **kwargs): self.relationship_index = 0 self.relationships = ["first", "second", "third", "fourth"] diff --git a/extensions/python/tests/features/resources/SetRecordField.py b/extensions/python/tests/features/resources/SetRecordField.py index dbb3a2ae64..80f57bb42d 100644 --- a/extensions/python/tests/features/resources/SetRecordField.py +++ b/extensions/python/tests/features/resources/SetRecordField.py @@ -13,26 +13,29 @@ # See the License for the specific language governing permissions and # limitations under the License. -from nifiapi.properties import PropertyDescriptor -from nifiapi.properties import StandardValidators -from nifiapi.properties import ExpressionLanguageScope -from nifiapi.recordtransform import RecordTransformResult -from nifiapi.recordtransform import RecordTransform +from typing import ClassVar + +from nifiapi.properties import ( + ExpressionLanguageScope, + PropertyDescriptor, + StandardValidators, +) +from nifiapi.recordtransform import RecordTransform, RecordTransformResult class SetRecordField(RecordTransform): class Java: - implements = ['org.apache.nifi.python.processor.RecordTransform'] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.RecordTransform"] class ProcessorDetails: - version = '0.0.1-SNAPSHOT' + version = "0.0.1-SNAPSHOT" def __init__(self, **kwargs): super().__init__() def transform(self, context, record, schema, attributemap): # Update dictionary based on the dynamic properties provided by user - for key in context.getProperties().keys(): + for key in context.getProperties(): if not key.dynamic: continue @@ -40,18 +43,18 @@ def transform(self, context, record, schema, attributemap): record[propname] = context.getProperty(propname).evaluateAttributeExpressions(attributemap).getValue() # Determine the partition - if 'group' in record: - partition = {'group': record['group']} + if "group" in record: + partition = {"group": record["group"]} else: partition = None # Return the result - return RecordTransformResult(record=record, relationship='success', partition=partition) + return RecordTransformResult(record=record, relationship="success", partition=partition) def getDynamicPropertyDescriptor(self, name): return PropertyDescriptor( name=name, description="Specifies the value to set for the '" + name + "' field", expression_language_scope=ExpressionLanguageScope.FLOWFILE_ATTRIBUTES, - validators=[StandardValidators.ALWAYS_VALID] + validators=[StandardValidators.ALWAYS_VALID], ) diff --git a/extensions/python/tests/features/resources/SingleThreadedSleepForever.py b/extensions/python/tests/features/resources/SingleThreadedSleepForever.py index 93c23e382b..d69b6f7187 100644 --- a/extensions/python/tests/features/resources/SingleThreadedSleepForever.py +++ b/extensions/python/tests/features/resources/SingleThreadedSleepForever.py @@ -14,19 +14,21 @@ # limitations under the License. import time -from nifiapi.flowfilesource import FlowFileSource +from typing import ClassVar + from nifiapi.decorators import trigger_serially +from nifiapi.flowfilesource import FlowFileSource @trigger_serially class SingleThreadedSleepForever(FlowFileSource): class Java: - implements = ['org.apache.nifi.python.processor.FlowFileSource'] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.FlowFileSource"] class ProcessorDetails: - version = '2.0.0-snapshot' - description = '''NiFi-style sleep forever python processor (single-threaded).''' - tags = ['test', 'python', 'sleep'] + version = "2.0.0-snapshot" + description = """NiFi-style sleep forever python processor (single-threaded).""" + tags: ClassVar[list] = ["test", "python", "sleep"] def __init__(self, **kwargs): pass diff --git a/extensions/python/tests/features/resources/SleepForever.py b/extensions/python/tests/features/resources/SleepForever.py index ee3d034ee9..a4cfb6b678 100644 --- a/extensions/python/tests/features/resources/SleepForever.py +++ b/extensions/python/tests/features/resources/SleepForever.py @@ -14,17 +14,19 @@ # limitations under the License. import time +from typing import ClassVar + from nifiapi.flowfilesource import FlowFileSource class SleepForever(FlowFileSource): class Java: - implements = ['org.apache.nifi.python.processor.FlowFileSource'] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.FlowFileSource"] class ProcessorDetails: - version = '2.0.0-snapshot' - description = '''NiFi-style sleep forever python processor.''' - tags = ['test', 'python', 'sleep'] + version = "2.0.0-snapshot" + description = """NiFi-style sleep forever python processor.""" + tags: ClassVar[list] = ["test", "python", "sleep"] def __init__(self, **kwargs): pass diff --git a/extensions/python/tests/features/resources/SpecialPropertyTypeChecker.py b/extensions/python/tests/features/resources/SpecialPropertyTypeChecker.py index f9b0a35aa2..098768f2bf 100644 --- a/extensions/python/tests/features/resources/SpecialPropertyTypeChecker.py +++ b/extensions/python/tests/features/resources/SpecialPropertyTypeChecker.py @@ -13,37 +13,40 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.flowfiletransform import FlowFileTransform, FlowFileTransformResult -from nifiapi.properties import PropertyDescriptor, TimeUnit, DataUnit +from nifiapi.properties import DataUnit, PropertyDescriptor, TimeUnit class SpecialPropertyTypeChecker(FlowFileTransform): """ Checks if custom property types are working as expected """ + TIME_PERIOD_PROPERTY = PropertyDescriptor( name="Time Period Property", description="Dummy property that should be a time period", default_value="2 hours", - required=True + required=True, ) DATA_SIZE_PROPERTY = PropertyDescriptor( name="Data Size Property", description="Dummy property that should be a data size", default_value="100 MB", - required=True + required=True, ) SSL_CONTEXT_PROPERTY = PropertyDescriptor( name="SSL Context Service", description="Dummy property that should be an SSL Context", controller_service_definition="SSLContextService", - required=True + required=True, ) - property_descriptors = [ + property_descriptors: ClassVar[list] = [ TIME_PERIOD_PROPERTY, DATA_SIZE_PROPERTY, - SSL_CONTEXT_PROPERTY + SSL_CONTEXT_PROPERTY, ] def __init__(self, **kwargs): diff --git a/extensions/python/tests/features/resources/TestStateManager.py b/extensions/python/tests/features/resources/TestStateManager.py index b5d657176d..b092064a11 100644 --- a/extensions/python/tests/features/resources/TestStateManager.py +++ b/extensions/python/tests/features/resources/TestStateManager.py @@ -13,18 +13,20 @@ # See the License for the specific language governing permissions and # limitations under the License. +from typing import ClassVar + from nifiapi.componentstate import Scope from nifiapi.flowfilesource import FlowFileSource, FlowFileSourceResult class TestStateManager(FlowFileSource): class Java: - implements = ['org.apache.nifi.python.processor.FlowFileSource'] + implements: ClassVar[list] = ["org.apache.nifi.python.processor.FlowFileSource"] class ProcessorDetails: - version = '0.0.1-SNAPSHOT' - description = '''A Python source processor that uses StateManager.''' - tags = ['text', 'test', 'python', 'source'] + version = "0.0.1-SNAPSHOT" + description = """A Python source processor that uses StateManager.""" + tags: ClassVar[list] = ["text", "test", "python", "source"] def __init__(self, **kwargs): pass @@ -37,12 +39,16 @@ def create(self, context): state = state_manager.getState(Scope.CLUSTER) old_value = state.get("state_key") if not old_value: - new_state = {'state_key': '1'} + new_state = {"state_key": "1"} state_manager.setState(new_state, Scope.CLUSTER) - elif old_value == '1': - new_state = {'state_key': '2'} + elif old_value == "1": + new_state = {"state_key": "2"} state_manager.replace(state, new_state, Scope.CLUSTER) else: state_manager.clear(Scope.CLUSTER) - return FlowFileSourceResult(relationship='success', attributes=state.toMap(), contents='Output FlowFile Contents') + return FlowFileSourceResult( + relationship="success", + attributes=state.toMap(), + contents="Output FlowFile Contents", + ) diff --git a/extensions/python/tests/features/resources/multiplierutils.py b/extensions/python/tests/features/resources/multiplierutils.py index 7b0bafa67a..005cbd252d 100644 --- a/extensions/python/tests/features/resources/multiplierutils.py +++ b/extensions/python/tests/features/resources/multiplierutils.py @@ -13,5 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. + def double(number): return number * 2 diff --git a/extensions/python/tests/features/resources/subtractutils.py b/extensions/python/tests/features/resources/subtractutils.py index 854a1d3c06..fd7bc076c2 100644 --- a/extensions/python/tests/features/resources/subtractutils.py +++ b/extensions/python/tests/features/resources/subtractutils.py @@ -13,5 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. + def minus_ten(number): return number - 10 diff --git a/extensions/python/tests/features/steps/steps.py b/extensions/python/tests/features/steps/steps.py index 6b0dc4a479..5bb23c3fa3 100644 --- a/extensions/python/tests/features/steps/steps.py +++ b/extensions/python/tests/features/steps/steps.py @@ -13,15 +13,18 @@ # See the License for the specific language governing permissions and # limitations under the License. import os -import docker -from behave import given -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.core.minifi_test_context import MinifiTestContext +from behave import given from minifi_behave.containers.docker_image_builder import DockerImageBuilder +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) + +import docker class PythonWithDependenciesOptions: @@ -31,11 +34,13 @@ class PythonWithDependenciesOptions: def build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(context, python_option): - minifi_tag_prefix = os.environ['MINIFI_TAG_PREFIX'] if 'MINIFI_TAG_PREFIX' in os.environ else "" + minifi_tag_prefix = os.environ.get("MINIFI_TAG_PREFIX", "") client: docker.DockerClient = docker.from_env() - is_fhs = 'MINIFI_INSTALLATION_TYPE=FHS' in str(client.images.get(context.minifi_container_image).history()) - minifi_python_dir_path = '/var/lib/nifi-minifi-cpp/minifi-python' if is_fhs else '/opt/minifi/minifi-current/minifi-python' - minifi_python_venv_parent = '/var/lib/nifi-minifi-cpp' if is_fhs else '/opt/minifi/minifi-current' + is_fhs = "MINIFI_INSTALLATION_TYPE=FHS" in str(client.images.get(context.minifi_container_image).history()) + minifi_python_dir_path = ( + "/var/lib/nifi-minifi-cpp/minifi-python" if is_fhs else "/opt/minifi/minifi-current/minifi-python" + ) + minifi_python_venv_parent = "/var/lib/nifi-minifi-cpp" if is_fhs else "/opt/minifi/minifi-current" parse_document_url = "https://raw.githubusercontent.com/apache/nifi-python-extensions/refs/heads/main/src/extensions/chunking/ParseDocument.py" chunk_document_url = "https://raw.githubusercontent.com/apache/nifi-python-extensions/refs/heads/main/src/extensions/chunking/ChunkDocument.py" pip3_install_command = "" @@ -45,29 +50,39 @@ def build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(contex # /class ProcessorDetails:/,/^$/: Do the following between 'class ProcessorDetails:' and the first empty line (so we don't modify other PropertyDescriptor blocks below) # /^\s*dependencies\s*=/,/\]\s*$/: Do the following between 'dependencies =' at the start of a line, and ']' at the end of a line # d: Delete line - parse_document_sed_cmd = 'sed -i "/class ProcessorDetails:/,/^$/{{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}}" {minifi_python_dir}/nifi_python_processors/ParseDocument.py && \\'.format(minifi_python_dir=minifi_python_dir_path) - chunk_document_sed_cmd = 'sed -i "/class ProcessorDetails:/,/^$/{{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}}" {minifi_python_dir}/nifi_python_processors/ChunkDocument.py && \\'.format(minifi_python_dir=minifi_python_dir_path) + parse_document_sed_cmd = f'sed -i "/class ProcessorDetails:/,/^$/{{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}}" {minifi_python_dir_path}/nifi_python_processors/ParseDocument.py && \\' + chunk_document_sed_cmd = f'sed -i "/class ProcessorDetails:/,/^$/{{/^\\s*dependencies\\s*=/,/\\]\\s*$/d}}" {minifi_python_dir_path}/nifi_python_processors/ChunkDocument.py && \\' if python_option == PythonWithDependenciesOptions.SYSTEM_INSTALLED_PACKAGES: - if not minifi_tag_prefix or "bookworm" in minifi_tag_prefix or "noble" in minifi_tag_prefix or "trixie" in minifi_tag_prefix: + if ( + not minifi_tag_prefix + or "bookworm" in minifi_tag_prefix + or "noble" in minifi_tag_prefix + or "trixie" in minifi_tag_prefix + ): additional_cmd = "RUN pip3 install --break-system-packages 'langchain<=0.17.0'" else: additional_cmd = "RUN pip3 install 'langchain<=0.17.0'" elif python_option == PythonWithDependenciesOptions.REQUIREMENTS_FILE: - requirements_install_command = "echo 'langchain<=0.17.0' > {minifi_python_dir}/nifi_python_processors/requirements.txt && \\".format(minifi_python_dir=minifi_python_dir_path) + requirements_install_command = ( + f"echo 'langchain<=0.17.0' > {minifi_python_dir_path}/nifi_python_processors/requirements.txt && \\" + ) elif python_option == PythonWithDependenciesOptions.INLINE_DEFINED_PACKAGES: - parse_document_sed_cmd = parse_document_sed_cmd[:-2] + ' sed -i "s/langchain==[0-9.]\\+/langchain<=0.17.0/" {minifi_python_dir}/nifi_python_processors/ParseDocument.py && \\'.format(minifi_python_dir=minifi_python_dir_path) - chunk_document_sed_cmd = 'sed -i "s/\\[\\\'langchain\\\'\\]/\\[\\\'langchain<=0.17.0\\\'\\]/" {minifi_python_dir}/nifi_python_processors/ChunkDocument.py && \\'.format(minifi_python_dir=minifi_python_dir_path) + parse_document_sed_cmd = ( + parse_document_sed_cmd[:-2] + + f' sed -i "s/langchain==[0-9.]\\+/langchain<=0.17.0/" {minifi_python_dir_path}/nifi_python_processors/ParseDocument.py && \\' + ) + chunk_document_sed_cmd = f"sed -i \"s/\\[\\'langchain\\'\\]/\\[\\'langchain<=0.17.0\\'\\]/\" {minifi_python_dir_path}/nifi_python_processors/ChunkDocument.py && \\" if not minifi_tag_prefix: pip3_install_command = "RUN apk --update --no-cache add py3-pip" - dockerfile = """\ -FROM {base_image} + dockerfile = f"""\ +FROM {context.minifi_container_image} USER root {pip3_install_command} {additional_cmd} USER minificpp -RUN wget {parse_document_url} --directory-prefix={minifi_python_dir}/nifi_python_processors && \\ - wget {chunk_document_url} --directory-prefix={minifi_python_dir}/nifi_python_processors && \\ - echo 'langchain<=0.17.0' > {minifi_python_dir}/nifi_python_processors/requirements.txt && \\ +RUN wget {parse_document_url} --directory-prefix={minifi_python_dir_path}/nifi_python_processors && \\ + wget {chunk_document_url} --directory-prefix={minifi_python_dir_path}/nifi_python_processors && \\ + echo 'langchain<=0.17.0' > {minifi_python_dir_path}/nifi_python_processors/requirements.txt && \\ {requirements_install_command} {parse_document_sed_cmd} {chunk_document_sed_cmd} @@ -75,16 +90,7 @@ def build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(contex python3 -m venv {minifi_python_venv_parent}/venv-with-langchain && \\ . {minifi_python_venv_parent}/venv-with-langchain/bin/activate && python3 -m pip install --no-cache-dir "langchain<=0.17.0" && \\ deactivate - """.format(base_image=context.minifi_container_image, - pip3_install_command=pip3_install_command, - parse_document_url=parse_document_url, - chunk_document_url=chunk_document_url, - additional_cmd=additional_cmd, - requirements_install_command=requirements_install_command, - parse_document_sed_cmd=parse_document_sed_cmd, - chunk_document_sed_cmd=chunk_document_sed_cmd, - minifi_python_dir=minifi_python_dir_path, - minifi_python_venv_parent=minifi_python_venv_parent) + """ builder = DockerImageBuilder( image_tag=python_option, dockerfile_content=dockerfile, @@ -101,22 +107,45 @@ def add_example_minifi_python_processors(context: MinifiTestContext): @given("python with langchain is installed on the MiNiFi agent {install_mode}") def install_python_with_langchain(context, install_mode): client: docker.DockerClient = docker.from_env() - is_fhs = 'MINIFI_INSTALLATION_TYPE=FHS' in str(client.images.get(context.minifi_container_image).history()) - minifi_python_venv_parent = '/var/lib/nifi-minifi-cpp' if is_fhs else '/opt/minifi/minifi-current' + is_fhs = "MINIFI_INSTALLATION_TYPE=FHS" in str(client.images.get(context.minifi_container_image).history()) + minifi_python_venv_parent = "/var/lib/nifi-minifi-cpp" if is_fhs else "/opt/minifi/minifi-current" if install_mode == "with required python packages": - build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(context, PythonWithDependenciesOptions.SYSTEM_INSTALLED_PACKAGES) - context.get_or_create_default_minifi_container().set_property("nifi.python.install.packages.automatically", "false") + build_minifi_cpp_image_with_nifi_python_processors_using_dependencies( + context, PythonWithDependenciesOptions.SYSTEM_INSTALLED_PACKAGES + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.python.install.packages.automatically", "false" + ) elif install_mode == "with a pre-created virtualenv": - build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(context, PythonWithDependenciesOptions.REQUIREMENTS_FILE) - context.get_or_create_default_minifi_container().set_property("nifi.python.virtualenv.directory", f"{minifi_python_venv_parent}/venv") - context.get_or_create_default_minifi_container().set_property("nifi.python.install.packages.automatically", "true") + build_minifi_cpp_image_with_nifi_python_processors_using_dependencies( + context, PythonWithDependenciesOptions.REQUIREMENTS_FILE + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.python.virtualenv.directory", f"{minifi_python_venv_parent}/venv" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.python.install.packages.automatically", "true" + ) elif install_mode == "with a pre-created virtualenv containing the required python packages": - build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(context, PythonWithDependenciesOptions.REQUIREMENTS_FILE) - context.get_or_create_default_minifi_container().set_property("nifi.python.virtualenv.directory", f"{minifi_python_venv_parent}/venv-with-langchain") - context.get_or_create_default_minifi_container().set_property("nifi.python.install.packages.automatically", "false") + build_minifi_cpp_image_with_nifi_python_processors_using_dependencies( + context, PythonWithDependenciesOptions.REQUIREMENTS_FILE + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.python.virtualenv.directory", + f"{minifi_python_venv_parent}/venv-with-langchain", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.python.install.packages.automatically", "false" + ) elif install_mode == "using inline defined Python dependencies to install packages": - build_minifi_cpp_image_with_nifi_python_processors_using_dependencies(context, PythonWithDependenciesOptions.INLINE_DEFINED_PACKAGES) - context.get_or_create_default_minifi_container().set_property("nifi.python.virtualenv.directory", f"{minifi_python_venv_parent}/venv") - context.get_or_create_default_minifi_container().set_property("nifi.python.install.packages.automatically", "true") + build_minifi_cpp_image_with_nifi_python_processors_using_dependencies( + context, PythonWithDependenciesOptions.INLINE_DEFINED_PACKAGES + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.python.virtualenv.directory", f"{minifi_python_venv_parent}/venv" + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.python.install.packages.automatically", "true" + ) else: - raise Exception("Unknown python install mode.") + raise RuntimeError("Unknown python install mode.") diff --git a/extensions/python/tests/test_python_scripts/foo_bar_processor.py b/extensions/python/tests/test_python_scripts/foo_bar_processor.py index 199a5ce898..b32c39bb24 100644 --- a/extensions/python/tests/test_python_scripts/foo_bar_processor.py +++ b/extensions/python/tests/test_python_scripts/foo_bar_processor.py @@ -15,8 +15,8 @@ # See the License for the specific language governing permissions and # limitations under the License. # -import foo import bar +import foo def describe(processor): diff --git a/extensions/python/tests/test_python_scripts/non_transferring_processor.py b/extensions/python/tests/test_python_scripts/non_transferring_processor.py index afb83d4cea..3897cb0c9b 100644 --- a/extensions/python/tests/test_python_scripts/non_transferring_processor.py +++ b/extensions/python/tests/test_python_scripts/non_transferring_processor.py @@ -23,7 +23,7 @@ def describe(processor): def onTrigger(context, session): flow_file = session.get() - log.info('Vrrm, vrrrm, processor is running, vrrrm!!') + log.info("Vrrm, vrrrm, processor is running, vrrrm!!") if flow_file is not None: - log.info('created flow file: %s' % flow_file.getAttribute('filename')) + log.info("created flow file: {}".format(flow_file.getAttribute("filename"))) diff --git a/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_failure.py b/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_failure.py index cad1d8e02a..99a6b0cafd 100644 --- a/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_failure.py +++ b/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_failure.py @@ -23,8 +23,8 @@ def describe(processor): def onTrigger(context, session): flow_file = session.get() - log.info('Vrrm, vrrrm, processor is running, vrrrm!!') + log.info("Vrrm, vrrrm, processor is running, vrrrm!!") if flow_file is not None: - log.info('created flow file: %s' % flow_file.getAttribute('filename')) + log.info("created flow file: {}".format(flow_file.getAttribute("filename"))) session.transfer(flow_file, REL_FAILURE) diff --git a/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_success.py b/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_success.py index 5ef7f5fbbf..b2d5235003 100644 --- a/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_success.py +++ b/extensions/python/tests/test_python_scripts/passthrough_processor_transfering_to_success.py @@ -23,8 +23,8 @@ def describe(processor): def onTrigger(context, session): flow_file = session.get() - log.info('Vrrm, vrrrm, processor is running, vrrrm!!') + log.info("Vrrm, vrrrm, processor is running, vrrrm!!") if flow_file is not None: - log.info('created flow file: %s' % flow_file.getAttribute('filename')) + log.info("created flow file: {}".format(flow_file.getAttribute("filename"))) session.transfer(flow_file, REL_SUCCESS) diff --git a/extensions/python/tests/test_python_scripts/stateful_processor.py b/extensions/python/tests/test_python_scripts/stateful_processor.py index 12aaecca9d..d748294719 100644 --- a/extensions/python/tests/test_python_scripts/stateful_processor.py +++ b/extensions/python/tests/test_python_scripts/stateful_processor.py @@ -21,22 +21,21 @@ def describe(processor): state = 0 -class WriteCallback(object): +class WriteCallback: def process(self, output_stream): global state - new_content = str(state).encode('utf-8') + new_content = str(state).encode("utf-8") output_stream.write(new_content) state = state + 1 return len(new_content) def onTrigger(context, session): - global state - log.info('Vrrm, vrrrm, processor is running, vrrrm!!') + log.info("Vrrm, vrrrm, processor is running, vrrrm!!") # flow_file = session.get() flow_file = session.create() flow_file.setAttribute("filename", str(state)) - log.info('created flow file: %s' % flow_file.getAttribute('filename')) + log.info("created flow file: {}".format(flow_file.getAttribute("filename"))) if flow_file is not None: session.write(flow_file, WriteCallback()) diff --git a/extensions/splunk/tests/features/containers/splunk_container.py b/extensions/splunk/tests/features/containers/splunk_container.py index f1cb7b2f5e..de536f6a43 100644 --- a/extensions/splunk/tests/features/containers/splunk_container.py +++ b/extensions/splunk/tests/features/containers/splunk_container.py @@ -16,22 +16,28 @@ import json from minifi_behave.containers.container_linux import LinuxContainer -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.helpers import wait_for_condition, retry_check from minifi_behave.containers.file import File -from minifi_behave.core.ssl_utils import make_server_cert, dump_cert, dump_key +from minifi_behave.core.helpers import retry_check, wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert class SplunkContainer(LinuxContainer): IMAGE = "splunk/splunk:9.2.1-patch2" def __init__(self, test_context: MinifiTestContext): - super().__init__(SplunkContainer.IMAGE, f"splunk-{test_context.scenario_id}", test_context.network) + super().__init__( + SplunkContainer.IMAGE, + f"splunk-{test_context.scenario_id}", + test_context.network, + ) self.user = None - self.environment = ["SPLUNK_LICENSE_URI=Free", - "SPLUNK_START_ARGS=--accept-license", - "SPLUNK_PASSWORD=splunkadmin"] + self.environment = [ + "SPLUNK_LICENSE_URI=Free", + "SPLUNK_START_ARGS=--accept-license", + "SPLUNK_PASSWORD=splunkadmin", + ] splunk_config_content = """ splunk: @@ -41,14 +47,35 @@ def __init__(self, test_context: MinifiTestContext): port: 8088 token: 176fae97-f59d-4f08-939a-aa6a543f2485 """ - self.files.append(File("/tmp/defaults/default.yml", splunk_config_content, mode="rw", permissions=0o644)) + self.files.append( + File( + "/tmp/defaults/default.yml", + splunk_config_content, + mode="rw", + permissions=0o644, + ) + ) - splunk_cert, splunk_key = make_server_cert(self.container_name, test_context.root_ca_cert, test_context.root_ca_key) + splunk_cert, splunk_key = make_server_cert( + self.container_name, test_context.root_ca_cert, test_context.root_ca_key + ) splunk_cert_content = dump_cert(splunk_cert) splunk_key_content = dump_key(splunk_key) root_ca_content = dump_cert(test_context.root_ca_cert) - self.files.append(File("/opt/splunk/etc/auth/splunk_cert.pem", splunk_cert_content.decode() + splunk_key_content.decode() + root_ca_content.decode(), permissions=0o644)) - self.files.append(File("/opt/splunk/etc/auth/root_ca.pem", root_ca_content.decode(), permissions=0o644)) + self.files.append( + File( + "/opt/splunk/etc/auth/splunk_cert.pem", + splunk_cert_content.decode() + splunk_key_content.decode() + root_ca_content.decode(), + permissions=0o644, + ) + ) + self.files.append( + File( + "/opt/splunk/etc/auth/root_ca.pem", + root_ca_content.decode(), + permissions=0o644, + ) + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -57,18 +84,39 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=300, bail_condition=lambda: False, - context=context) + context=context, + ) @retry_check() def check_splunk_event(self, query: str) -> bool: - (code, output) = self.exec_run(["sudo", "/opt/splunk/bin/splunk", "search", query, "-auth", "admin:splunkadmin"]) + (code, output) = self.exec_run( + [ + "sudo", + "/opt/splunk/bin/splunk", + "search", + query, + "-auth", + "admin:splunkadmin", + ] + ) if code != 0: return False return query in output.decode("utf-8") @retry_check() def check_splunk_event_with_attributes(self, query: str, attributes: dict[str, str]) -> bool: - (code, output) = self.exec_run(["sudo", "/opt/splunk/bin/splunk", "search", query, "-output", "json", "-auth", "admin:splunkadmin"]) + (code, output) = self.exec_run( + [ + "sudo", + "/opt/splunk/bin/splunk", + "search", + query, + "-output", + "json", + "-auth", + "admin:splunkadmin", + ] + ) if code != 0: return False result_lines = output.splitlines() @@ -79,39 +127,56 @@ def check_splunk_event_with_attributes(self, query: str, attributes: dict[str, s continue if "result" not in result_line_json: continue - if "host" in attributes: - if result_line_json["result"]["host"] != attributes["host"]: - continue - if "source" in attributes: - if result_line_json["result"]["source"] != attributes["source"]: - continue - if "sourcetype" in attributes: - if result_line_json["result"]["sourcetype"] != attributes["sourcetype"]: - continue - if "index" in attributes: - if result_line_json["result"]["index"] != attributes["index"]: - continue + if "host" in attributes and result_line_json["result"]["host"] != attributes["host"]: + continue + if "source" in attributes and result_line_json["result"]["source"] != attributes["source"]: + continue + if "sourcetype" in attributes and result_line_json["result"]["sourcetype"] != attributes["sourcetype"]: + continue + if "index" in attributes and result_line_json["result"]["index"] != attributes["index"]: + continue return True return False def enable_splunk_hec_indexer(self, hec_name: str): - (code, _) = self.exec_run(["sudo", - "/opt/splunk/bin/splunk", "http-event-collector", - "update", hec_name, - "-uri", "https://localhost:8089", - "-use-ack", "1", - "-disabled", "0", - "-auth", "admin:splunkadmin"]) + (code, _) = self.exec_run( + [ + "sudo", + "/opt/splunk/bin/splunk", + "http-event-collector", + "update", + hec_name, + "-uri", + "https://localhost:8089", + "-use-ack", + "1", + "-disabled", + "0", + "-auth", + "admin:splunkadmin", + ] + ) return code == 0 def enable_splunk_hec_ssl(self): - (code, _) = self.exec_run(["sudo", - "/opt/splunk/bin/splunk", "http-event-collector", - "update", - "-uri", "https://localhost:8089", - "-enable-ssl", "1", - "-server-cert", "/opt/splunk/etc/auth/splunk_cert.pem", - "-ca-cert-file", "/opt/splunk/etc/auth/root_ca.pem", - "-require-client-cert", "1", - "-auth", "admin:splunkadmin"]) + (code, _) = self.exec_run( + [ + "sudo", + "/opt/splunk/bin/splunk", + "http-event-collector", + "update", + "-uri", + "https://localhost:8089", + "-enable-ssl", + "1", + "-server-cert", + "/opt/splunk/etc/auth/splunk_cert.pem", + "-ca-cert-file", + "/opt/splunk/etc/auth/root_ca.pem", + "-require-client-cert", + "1", + "-auth", + "admin:splunkadmin", + ] + ) return code == 0 diff --git a/extensions/splunk/tests/features/environment.py b/extensions/splunk/tests/features/environment.py index 4598a1f0e6..52694dc0b7 100644 --- a/extensions/splunk/tests/features/environment.py +++ b/extensions/splunk/tests/features/environment.py @@ -14,11 +14,11 @@ # limitations under the License. import platform -import docker from containers.splunk_container import SplunkContainer -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + +import docker def before_feature(context, feature): diff --git a/extensions/splunk/tests/features/steps/steps.py b/extensions/splunk/tests/features/steps/steps.py index c89a329556..e24063e5a9 100644 --- a/extensions/splunk/tests/features/steps/steps.py +++ b/extensions/splunk/tests/features/steps/steps.py @@ -13,34 +13,42 @@ # See the License for the specific language governing permissions and # limitations under the License. from behave import step, then - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from containers.splunk_container import SplunkContainer from minifi_behave.core.helpers import log_due_to_failure from minifi_behave.core.minifi_test_context import MinifiTestContext -from containers.splunk_container import SplunkContainer +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) @step("a Splunk HEC is set up and running") def setup_splunk_hec(context: MinifiTestContext): context.containers["splunk"] = SplunkContainer(context) assert context.containers["splunk"].deploy(context) - assert context.containers["splunk"].enable_splunk_hec_indexer('splunk_hec_token') or context.containers["splunk"].log_app_output() + assert ( + context.containers["splunk"].enable_splunk_hec_indexer("splunk_hec_token") + or context.containers["splunk"].log_app_output() + ) -@then('an event is registered in Splunk HEC with the content \"{content}\"') +@then('an event is registered in Splunk HEC with the content "{content}"') def verify_splunk_event_content(context, content): assert context.containers["splunk"].check_splunk_event(content) or log_due_to_failure(context) -@then('an event is registered in Splunk HEC with the content \"{content}\" with \"{source}\" set as source and \"{source_type}\" set as sourcetype and \"{host}\" set as host') +@then( + 'an event is registered in Splunk HEC with the content "{content}" with "{source}" set as source and "{source_type}" set as sourcetype and "{host}" set as host' +) def verify_splunk_event_with_attributes(context, content, source, source_type, host): attr = {"source": source, "sourcetype": source_type, "host": host} assert context.containers["splunk"].check_splunk_event_with_attributes(content, attr) or log_due_to_failure(context) -@step("SSL is enabled for the Splunk HEC and the SSL context service is set up for PutSplunkHTTP and QuerySplunkIndexingStatus") +@step( + "SSL is enabled for the Splunk HEC and the SSL context service is set up for PutSplunkHTTP and QuerySplunkIndexingStatus" +) def enable_splunk_hec_ssl(context): assert context.containers["splunk"].enable_splunk_hec_ssl() or context.containers["splunk"].log_app_output() diff --git a/extensions/sql/tests/features/containers/postgress_server_container.py b/extensions/sql/tests/features/containers/postgress_server_container.py index 9a9ec6f6ff..a8c57c7eeb 100644 --- a/extensions/sql/tests/features/containers/postgress_server_container.py +++ b/extensions/sql/tests/features/containers/postgress_server_container.py @@ -17,18 +17,22 @@ import logging from textwrap import dedent + from minifi_behave.containers.container_linux import LinuxContainer from minifi_behave.containers.docker_image_builder import DockerImageBuilder from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +logger = logging.getLogger(__name__) + class PostgresContainer(LinuxContainer): IMAGE = "postgres:17.4" def __init__(self, context): - dockerfile = dedent("""\ - FROM {base_image} + dockerfile = dedent( + f"""\ + FROM {PostgresContainer.IMAGE} RUN mkdir -p /docker-entrypoint-initdb.d RUN echo "#!/bin/bash" > /docker-entrypoint-initdb.d/init-user-db.sh && \ echo "set -e" >> /docker-entrypoint-initdb.d/init-user-db.sh && \ @@ -41,26 +45,29 @@ def __init__(self, context): echo " INSERT INTO test_table2 (int_col, \\"tExT_Col\\") VALUES (5, 'ApPlE');" >> /docker-entrypoint-initdb.d/init-user-db.sh && \ echo " INSERT INTO test_table2 (int_col, \\"tExT_Col\\") VALUES (6, 'BaNaNa');" >> /docker-entrypoint-initdb.d/init-user-db.sh && \ echo "EOSQL" >> /docker-entrypoint-initdb.d/init-user-db.sh - """.format(base_image=PostgresContainer.IMAGE)) - builder = DockerImageBuilder( - image_tag="minifi-postgres-server:latest", - dockerfile_content=dockerfile + """ ) + builder = DockerImageBuilder(image_tag="minifi-postgres-server:latest", dockerfile_content=dockerfile) builder.build() - super(PostgresContainer, self).__init__("minifi-postgres-server:latest", f"postgres-server-{context.scenario_id}", context.network) + super().__init__( + "minifi-postgres-server:latest", + f"postgres-server-{context.scenario_id}", + context.network, + ) self.environment = ["POSTGRES_PASSWORD=password"] def deploy(self, context: MinifiTestContext | None) -> bool: - super(PostgresContainer, self).deploy(context) + super().deploy(context) finished_str = "database system is ready to accept connections" return wait_for_condition( condition=lambda: finished_str in self.get_logs(), timeout_seconds=60, bail_condition=lambda: self.exited, - context=context) + context=context, + ) def check_query_results(self, query: str, number_of_rows: int) -> bool: (code, output) = self.exec_run(["psql", "-U", "postgres", "-c", query]) - logging.debug(f"check_query_results: {query} -> {output}") + logger.debug(f"check_query_results: {query} -> {output}") return code == 0 and (str(number_of_rows) + (" row" if number_of_rows == 1 else " rows")) in output diff --git a/extensions/sql/tests/features/environment.py b/extensions/sql/tests/features/environment.py index 5b5cf8151d..971fab598f 100644 --- a/extensions/sql/tests/features/environment.py +++ b/extensions/sql/tests/features/environment.py @@ -16,35 +16,42 @@ # import os -import docker from textwrap import dedent from containers.postgress_server_container import PostgresContainer from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario -from minifi_behave.core.hooks import get_minifi_container_image +from minifi_behave.core.hooks import ( + common_after_scenario, + common_before_scenario, + get_minifi_container_image, +) + +import docker # These hooks are executed by behave before and after each scenario # The common_before_scenario and common_after_scenario must be called for proper setup and tear down def before_all(context): - minifi_tag_prefix = os.environ['MINIFI_TAG_PREFIX'] if 'MINIFI_TAG_PREFIX' in os.environ else '' + minifi_tag_prefix = os.environ.get("MINIFI_TAG_PREFIX", "") if "rocky" in minifi_tag_prefix: install_sql_cmd = "dnf -y install postgresql-odbc" so_location = "psqlodbca.so" - elif "bullseye" in minifi_tag_prefix or "bookworm" in minifi_tag_prefix or "trixie" in minifi_tag_prefix: - install_sql_cmd = "apt -y install odbc-postgresql" - so_location = "/usr/lib/$(gcc -dumpmachine)/odbc/psqlodbca.so" - elif "jammy" in minifi_tag_prefix or "noble" in minifi_tag_prefix: + elif ( + "bullseye" in minifi_tag_prefix + or "bookworm" in minifi_tag_prefix + or "trixie" in minifi_tag_prefix + or "jammy" in minifi_tag_prefix + or "noble" in minifi_tag_prefix + ): install_sql_cmd = "apt -y install odbc-postgresql" so_location = "/usr/lib/$(gcc -dumpmachine)/odbc/psqlodbca.so" else: install_sql_cmd = "apk --update --no-cache add psqlodbc" so_location = "psqlodbca.so" - dockerfile = dedent("""\ - FROM {base_image} + dockerfile = dedent( + f"""\ + FROM {get_minifi_container_image()} USER root RUN {install_sql_cmd} RUN echo "[PostgreSQL ANSI]" > /odbcinst.ini.template && \ @@ -73,12 +80,9 @@ def before_all(context): echo "Password = password" >> /etc/odbc.ini && \ echo "Database = postgres" >> /etc/odbc.ini USER minificpp - """.format(base_image=get_minifi_container_image(), - install_sql_cmd=install_sql_cmd, so_location=so_location)) - builder = DockerImageBuilder( - image_tag="apacheminificpp-sql:behave", - dockerfile_content=dockerfile + """ ) + builder = DockerImageBuilder(image_tag="apacheminificpp-sql:behave", dockerfile_content=dockerfile) builder.build() docker.from_env().images.pull(PostgresContainer.IMAGE) diff --git a/extensions/sql/tests/features/steps/steps.py b/extensions/sql/tests/features/steps/steps.py index 92b95d22b0..516d7cd639 100644 --- a/extensions/sql/tests/features/steps/steps.py +++ b/extensions/sql/tests/features/steps/steps.py @@ -16,23 +16,27 @@ # import humanfriendly -from behave import step, given, then - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from behave import given, step, then +from containers.postgress_server_container import PostgresContainer from minifi_behave.core.helpers import wait_for_condition -from minifi_behave.minifi.controller_service import ControllerService from minifi_behave.core.minifi_test_context import MinifiTestContext -from containers.postgress_server_container import PostgresContainer +from minifi_behave.minifi.controller_service import ControllerService +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) -@given("an ODBCService is setup up for {processor_name} with the name \"{service_name}\"") +@given('an ODBCService is setup up for {processor_name} with the name "{service_name}"') def setup_odbc_service_for_processor(context: MinifiTestContext, processor_name: str, service_name: str): odb_service = ControllerService(class_name="ODBCService", service_name=service_name) postgres_server_hostname = f"postgres-server-{context.scenario_id}" - odb_service.add_property("Connection String", f"Driver={{PostgreSQL ANSI}};Server={postgres_server_hostname};Port=5432;Database=postgres;Uid=postgres;Pwd=password;") + odb_service.add_property( + "Connection String", + f"Driver={{PostgreSQL ANSI}};Server={postgres_server_hostname};Port=5432;Database=postgres;Uid=postgres;Pwd=password;", + ) context.get_or_create_default_minifi_container().flow_definition.controller_services.append(odb_service) processor = context.get_or_create_default_minifi_container().flow_definition.get_processor(processor_name) processor.add_property("DB Controller Service", "ODBCService") @@ -52,4 +56,5 @@ def verify_postgresql_query_results(context, query: str, rows: int, timeout_str: condition=lambda: postgres_container.check_query_results(query, int(rows)), timeout_seconds=timeout_seconds, bail_condition=lambda: postgres_container.exited, - context=context) + context=context, + ) diff --git a/extensions/standard-processors/tests/features/containers/diag_slave_container.py b/extensions/standard-processors/tests/features/containers/diag_slave_container.py index 68f78ae64f..d128118c86 100644 --- a/extensions/standard-processors/tests/features/containers/diag_slave_container.py +++ b/extensions/standard-processors/tests/features/containers/diag_slave_container.py @@ -23,6 +23,8 @@ from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +logger = logging.getLogger(__name__) + class DiagSlave(LinuxContainer): def __init__(self, test_context: MinifiTestContext): @@ -32,13 +34,14 @@ def __init__(self, test_context: MinifiTestContext): ENV PROTOCOL=tcp """) - builder = DockerImageBuilder( - image_tag="minifi-diag-slave-tcp:latest", - dockerfile_content=dockerfile - ) + builder = DockerImageBuilder(image_tag="minifi-diag-slave-tcp:latest", dockerfile_content=dockerfile) builder.build() - super().__init__("minifi-diag-slave-tcp:latest", f"diag-slave-tcp-{test_context.scenario_id}", test_context.network) + super().__init__( + "minifi-diag-slave-tcp:latest", + f"diag-slave-tcp-{test_context.scenario_id}", + test_context.network, + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -47,10 +50,10 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=30, bail_condition=lambda: self.exited, - context=context + context=context, ) def set_value_on_plc_with_modbus(self, modbus_cmd): (code, output) = self.exec_run(["modbus", "localhost", modbus_cmd]) - logging.info("Modbus command '%s' output: %s", modbus_cmd, output) + logger.info("Modbus command '%s' output: %s", modbus_cmd, output) return code == 0 diff --git a/extensions/standard-processors/tests/features/containers/syslog_container.py b/extensions/standard-processors/tests/features/containers/syslog_container.py index 8cbb117d14..b94adb77e7 100644 --- a/extensions/standard-processors/tests/features/containers/syslog_container.py +++ b/extensions/standard-processors/tests/features/containers/syslog_container.py @@ -20,5 +20,5 @@ class SyslogContainer(LinuxContainer): def __init__(self, protocol, context): - super(SyslogContainer, self).__init__("ubuntu:24.04", f"syslog-{protocol}-{context.scenario_id}", context.network) + super().__init__("ubuntu:24.04", f"syslog-{protocol}-{context.scenario_id}", context.network) self.command = f'/bin/bash -c "echo Syslog {protocol} client started; while true; do logger --{protocol} -n minifi-primary-{context.scenario_id} -P 514 sample_log; sleep 1; done"' diff --git a/extensions/standard-processors/tests/features/containers/tcp_client_container.py b/extensions/standard-processors/tests/features/containers/tcp_client_container.py index 37b8cbb318..5734c76c3c 100644 --- a/extensions/standard-processors/tests/features/containers/tcp_client_container.py +++ b/extensions/standard-processors/tests/features/containers/tcp_client_container.py @@ -27,7 +27,12 @@ def __init__(self, test_context: MinifiTestContext): f"nc minifi-primary-{test_context.scenario_id} 10254; " "sleep 1; done'" ) - super().__init__("alpine:3.17.3", f"tcp-client-{test_context.scenario_id}", test_context.network, cmd) + super().__init__( + "alpine:3.17.3", + f"tcp-client-{test_context.scenario_id}", + test_context.network, + cmd, + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -36,5 +41,5 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=30, bail_condition=lambda: self.exited, - context=context + context=context, ) diff --git a/extensions/standard-processors/tests/features/environment.py b/extensions/standard-processors/tests/features/environment.py index fb2b3b72c5..642c02647e 100644 --- a/extensions/standard-processors/tests/features/environment.py +++ b/extensions/standard-processors/tests/features/environment.py @@ -15,9 +15,9 @@ import os import platform +from minifi_behave.core.hooks import common_after_scenario, common_before_scenario + import docker -from minifi_behave.core.hooks import common_before_scenario -from minifi_behave.core.hooks import common_after_scenario # These hooks are executed by behave before and after each scenario # The common_before_scenario and common_after_scenario must be called for proper setup and tear down @@ -31,12 +31,12 @@ def before_feature(context, feature): def is_minifi_image_alpine_based(context): - if os.name == 'nt': + if os.name == "nt": return False client: docker.DockerClient = docker.from_env() container = client.containers.create( image=context.minifi_container_image, - command=['cat', '/etc/os-release'], + command=["cat", "/etc/os-release"], ) try: container.start() @@ -46,14 +46,13 @@ def is_minifi_image_alpine_based(context): container.remove(force=True) return False - return "alpine" in result.decode('utf-8').lower() + return "alpine" in result.decode("utf-8").lower() def before_scenario(context, scenario): common_before_scenario(context, scenario) - if "ALPINE_ONLY" in scenario.tags: - if not is_minifi_image_alpine_based(context): - scenario.skip("This scenario is only compatible with Alpine-based Minifi images") + if "ALPINE_ONLY" in scenario.tags and not is_minifi_image_alpine_based(context): + scenario.skip("This scenario is only compatible with Alpine-based Minifi images") def after_scenario(context, scenario): diff --git a/extensions/standard-processors/tests/features/steps/minifi_c2_server_container.py b/extensions/standard-processors/tests/features/steps/minifi_c2_server_container.py index a645499f67..d404cccfc8 100644 --- a/extensions/standard-processors/tests/features/steps/minifi_c2_server_container.py +++ b/extensions/standard-processors/tests/features/steps/minifi_c2_server_container.py @@ -15,32 +15,53 @@ # limitations under the License. # -import jks import os - -from cryptography.hazmat.primitives import serialization -from cryptography import x509 from pathlib import Path -from cryptography.hazmat.primitives.serialization import load_pem_private_key, pkcs12, BestAvailableEncryption +import jks +from cryptography import x509 +from cryptography.hazmat.primitives import serialization +from cryptography.hazmat.primitives.serialization import ( + BestAvailableEncryption, + load_pem_private_key, + pkcs12, +) from minifi_behave.containers.container_linux import LinuxContainer -from minifi_behave.core.helpers import wait_for_condition -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.ssl_utils import make_server_cert from minifi_behave.containers.file import File from minifi_behave.containers.host_file import HostFile -from minifi_behave.core.ssl_utils import dump_key, dump_cert +from minifi_behave.core.helpers import wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.core.ssl_utils import dump_cert, dump_key, make_server_cert class MinifiC2Server(LinuxContainer): def __init__(self, test_context: MinifiTestContext, ssl: bool = False): - super().__init__("apache/nifi-minifi-c2:1.27.0", f"minifi-c2-server-{test_context.scenario_id}", test_context.network) + super().__init__( + "apache/nifi-minifi-c2:1.27.0", + f"minifi-c2-server-{test_context.scenario_id}", + test_context.network, + ) if ssl: - c2_cert, c2_key = make_server_cert(f"minifi-c2-server-{test_context.scenario_id}", test_context.root_ca_cert, test_context.root_ca_key) - pke = jks.PrivateKeyEntry.new('c2-server-cert', [dump_cert(c2_cert, encoding_type=serialization.Encoding.DER)], dump_key(c2_key, encoding_type=serialization.Encoding.DER), 'rsa_raw') - server_keystore = jks.KeyStore.new('jks', [pke]) - server_keystore_content = server_keystore.saves('abcdefgh') - self.files.append(File("/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-keystore.jks", server_keystore_content, permissions=0o644)) + c2_cert, c2_key = make_server_cert( + f"minifi-c2-server-{test_context.scenario_id}", + test_context.root_ca_cert, + test_context.root_ca_key, + ) + pke = jks.PrivateKeyEntry.new( + "c2-server-cert", + [dump_cert(c2_cert, encoding_type=serialization.Encoding.DER)], + dump_key(c2_key, encoding_type=serialization.Encoding.DER), + "rsa_raw", + ) + server_keystore = jks.KeyStore.new("jks", [pke]) + server_keystore_content = server_keystore.saves("abcdefgh") + self.files.append( + File( + "/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-keystore.jks", + server_keystore_content, + permissions=0o644, + ) + ) private_key_pem = dump_key(test_context.root_ca_key) private_key = load_pem_private_key(private_key_pem, password=None) @@ -51,21 +72,48 @@ def __init__(self, test_context: MinifiTestContext, ssl: bool = False): key=private_key, cert=certificate, cas=None, - encryption_algorithm=BestAvailableEncryption(b'abcdefgh') + encryption_algorithm=BestAvailableEncryption(b"abcdefgh"), + ) + self.files.append( + File( + "/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-truststore.p12", + pkcs12_data, + permissions=0o644, + ) ) - self.files.append(File("/opt/minifi-c2/minifi-c2-current/certs/minifi-c2-server-truststore.p12", pkcs12_data, permissions=0o644)) - authorities_file_content = """ - CN=minifi-primary-{scenario_id}: + authorities_file_content = f""" + CN=minifi-primary-{test_context.scenario_id}: - CLASS_MINIFI_CPP - """.format(scenario_id=test_context.scenario_id) - self.files.append(File("/opt/minifi-c2/minifi-c2-current/conf/authorities.yaml", authorities_file_content, permissions=0o644)) + """ + self.files.append( + File( + "/opt/minifi-c2/minifi-c2-current/conf/authorities.yaml", + authorities_file_content, + permissions=0o644, + ) + ) resource_dir = Path(__file__).resolve().parent / "resources" / "minifi-c2-server" - self.host_files.append(HostFile("/opt/minifi-c2/minifi-c2-current/files/minifi-test-class/config.text.yml.v1", os.path.join(resource_dir, "config.yml"))) + self.host_files.append( + HostFile( + "/opt/minifi-c2/minifi-c2-current/files/minifi-test-class/config.text.yml.v1", + os.path.join(resource_dir, "config.yml"), + ) + ) if ssl: - self.host_files.append(HostFile("/opt/minifi-c2/minifi-c2-current/conf/authorizations.yaml", os.path.join(resource_dir, "authorizations.yaml"))) - self.host_files.append(HostFile("/opt/minifi-c2/minifi-c2-current/conf/c2.properties", os.path.join(resource_dir, "c2.properties"))) + self.host_files.append( + HostFile( + "/opt/minifi-c2/minifi-c2-current/conf/authorizations.yaml", + os.path.join(resource_dir, "authorizations.yaml"), + ) + ) + self.host_files.append( + HostFile( + "/opt/minifi-c2/minifi-c2-current/conf/c2.properties", + os.path.join(resource_dir, "c2.properties"), + ) + ) def deploy(self, context: MinifiTestContext | None) -> bool: super().deploy(context) @@ -74,5 +122,5 @@ def deploy(self, context: MinifiTestContext | None) -> bool: condition=lambda: finished_str in self.get_logs(), timeout_seconds=60, bail_condition=lambda: self.exited, - context=context + context=context, ) diff --git a/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py b/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py index f4dca56f89..ad49d6e11b 100644 --- a/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py +++ b/extensions/standard-processors/tests/features/steps/minifi_controller_steps.py @@ -15,77 +15,86 @@ # limitations under the License. # -from behave import given, when, then +from behave import given, then, when from minifi_behave.core.helpers import retry_check from minifi_behave.core.minifi_test_context import MinifiTestContext -@given('controller socket properties are set up') +@given("controller socket properties are set up") def setup_controller_socket_properties(context: MinifiTestContext): context.get_or_create_default_minifi_container().controller.set_controller_socket_properties() -@when('MiNiFi config is updated through MiNiFi controller') +@when("MiNiFi config is updated through MiNiFi controller") def update_minifi_config_via_controller(context: MinifiTestContext): context.get_or_create_default_minifi_container().controller.update_flow_config_through_controller() -@then('the updated config is persisted') +@then("the updated config is persisted") def verify_updated_config_is_persisted(context: MinifiTestContext): assert context.get_or_create_default_minifi_container().controller.updated_config_is_persisted() -@when('the {component} component is stopped through MiNiFi controller') +@when("the {component} component is stopped through MiNiFi controller") def stop_component_via_controller(context: MinifiTestContext, component: str): context.get_or_create_default_minifi_container().controller.stop_component_through_controller(component) -@when('the {component} component is started through MiNiFi controller') +@when("the {component} component is started through MiNiFi controller") def start_component_via_controller(context: MinifiTestContext, component: str): context.get_or_create_default_minifi_container().controller.start_component_through_controller(component) -@then('the {component} component is not running') +@then("the {component} component is not running") def verify_component_not_running(context: MinifiTestContext, component: str): assert not context.get_or_create_default_minifi_container().controller.is_component_running(component) -@then('the {component} component is running') +@then("the {component} component is running") def verify_component_is_running(context: MinifiTestContext, component: str): assert context.get_or_create_default_minifi_container().controller.is_component_running(component) -@then('connection \"{connection}\" can be seen through MiNiFi controller') +@then('connection "{connection}" can be seen through MiNiFi controller') def verify_connection_seen_via_controller(context: MinifiTestContext, connection: str): assert context.get_or_create_default_minifi_container().controller.connection_found_through_controller(connection) -@then('{connection_count:d} connections can be seen full through MiNiFi controller') +@then("{connection_count:d} connections can be seen full through MiNiFi controller") def verify_full_connection_count(context: MinifiTestContext, connection_count: int): assert context.get_or_create_default_minifi_container().controller.get_full_connection_count() == connection_count @retry_check(max_tries=5, retry_interval_seconds=1) -def check_connection_size_through_controller(context: MinifiTestContext, connection: str, size: int, max_size: int) -> bool: - return context.get_or_create_default_minifi_container().controller.get_connection_size(connection) == (size, max_size) +def check_connection_size_through_controller( + context: MinifiTestContext, connection: str, size: int, max_size: int +) -> bool: + return context.get_or_create_default_minifi_container().controller.get_connection_size(connection) == ( + size, + max_size, + ) -@then('connection \"{connection}\" has {size:d} size and {max_size:d} max size through MiNiFi controller') +@then('connection "{connection}" has {size:d} size and {max_size:d} max size through MiNiFi controller') def verify_connection_size_via_controller(context: MinifiTestContext, connection: str, size: int, max_size: int): assert check_connection_size_through_controller(context, connection, size, max_size) @retry_check(max_tries=10, retry_interval_seconds=1) -def manifest_can_be_retrieved_through_minifi_controller(context: MinifiTestContext) -> bool: +def manifest_can_be_retrieved_through_minifi_controller( + context: MinifiTestContext, +) -> bool: manifest = context.get_or_create_default_minifi_container().controller.get_manifest() - return '"agentManifest": {' in manifest and '"componentManifest": {' in manifest and '"agentType": "cpp"' in manifest + return ( + '"agentManifest": {' in manifest and '"componentManifest": {' in manifest and '"agentType": "cpp"' in manifest + ) -@then('manifest can be retrieved through MiNiFi controller') +@then("manifest can be retrieved through MiNiFi controller") def verify_manifest_retrieval_via_controller(context: MinifiTestContext): assert manifest_can_be_retrieved_through_minifi_controller(context) -@then('debug bundle can be retrieved through MiNiFi controller') +@then("debug bundle can be retrieved through MiNiFi controller") def verify_debug_bundle_retrieval_via_controller(context: MinifiTestContext): assert context.get_or_create_default_minifi_container().controller.create_debug_bundle() diff --git a/extensions/standard-processors/tests/features/steps/steps.py b/extensions/standard-processors/tests/features/steps/steps.py index 689a6aef99..b4469fbe9a 100644 --- a/extensions/standard-processors/tests/features/steps/steps.py +++ b/extensions/standard-processors/tests/features/steps/steps.py @@ -16,19 +16,19 @@ # import humanfriendly -from behave import step, then, given - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 -from minifi_behave.core.minifi_test_context import MinifiTestContext -from minifi_behave.core.helpers import wait_for_condition -from containers.syslog_container import SyslogContainer +from behave import given, step, then from containers.diag_slave_container import DiagSlave +from containers.syslog_container import SyslogContainer from containers.tcp_client_container import TcpClientContainer - from minifi_behave.containers.minifi_protocol import set_up_ssl_properties +from minifi_behave.core.helpers import wait_for_condition +from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) from minifi_c2_server_container import MinifiC2Server @@ -42,18 +42,21 @@ def setup_syslog_udp_client(context: MinifiTestContext): context.containers["syslog-udp"] = SyslogContainer("udp", context) -@step('there is an accessible PLC with modbus enabled') +@step("there is an accessible PLC with modbus enabled") def setup_plc_with_modbus(context: MinifiTestContext): modbus_container = context.containers["diag-slave-tcp"] = DiagSlave(context) assert modbus_container.deploy(context) -@step('PLC register has been set with {modbus_cmd} command') +@step("PLC register has been set with {modbus_cmd} command") def set_plc_register_with_command(context: MinifiTestContext, modbus_cmd: str): - assert context.containers["diag-slave-tcp"].set_value_on_plc_with_modbus(modbus_cmd) or context.containers["diag-slave-tcp"].log_app_output() + assert ( + context.containers["diag-slave-tcp"].set_value_on_plc_with_modbus(modbus_cmd) + or context.containers["diag-slave-tcp"].log_app_output() + ) -@step('a TCP client is set up to send a test TCP message to minifi') +@step("a TCP client is set up to send a test TCP message to minifi") def setup_tcp_client(context: MinifiTestContext): context.containers["tcp-client"] = TcpClientContainer(context) @@ -61,10 +64,22 @@ def setup_tcp_client(context: MinifiTestContext): @given("C2 is enabled in MiNiFi") def enable_c2_in_minifi(context: MinifiTestContext): context.get_or_create_default_minifi_container().set_property("nifi.c2.enable", "true") - context.get_or_create_default_minifi_container().set_property("nifi.c2.rest.url", f"http://minifi-c2-server-{context.scenario_id}:10090/c2/config/heartbeat") - context.get_or_create_default_minifi_container().set_property("nifi.c2.rest.url.ack", f"http://minifi-c2-server-{context.scenario_id}:10090/c2/config/acknowledge") - context.get_or_create_default_minifi_container().set_property("nifi.c2.flow.base.url", f"http://minifi-c2-server-{context.scenario_id}:10090/c2/config/") - context.get_or_create_default_minifi_container().set_property("nifi.c2.root.classes", "DeviceInfoNode,AgentInformation,FlowInformation,AssetInformation") + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.rest.url", + f"http://minifi-c2-server-{context.scenario_id}:10090/c2/config/heartbeat", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.rest.url.ack", + f"http://minifi-c2-server-{context.scenario_id}:10090/c2/config/acknowledge", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.flow.base.url", + f"http://minifi-c2-server-{context.scenario_id}:10090/c2/config/", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.root.classes", + "DeviceInfoNode,AgentInformation,FlowInformation,AssetInformation", + ) context.get_or_create_default_minifi_container().set_property("nifi.c2.full.heartbeat", "false") context.get_or_create_default_minifi_container().set_property("nifi.c2.agent.class", "minifi-test-class") context.get_or_create_default_minifi_container().set_property("nifi.c2.agent.identifier", "minifi-test-id") @@ -73,10 +88,22 @@ def enable_c2_in_minifi(context: MinifiTestContext): @given("ssl properties are set up for MiNiFi C2 server") def setup_ssl_properties_for_c2(context: MinifiTestContext): context.get_or_create_default_minifi_container().set_property("nifi.c2.enable", "true") - context.get_or_create_default_minifi_container().set_property("nifi.c2.rest.url", f"https://minifi-c2-server-{context.scenario_id}:10090/c2/config/heartbeat") - context.get_or_create_default_minifi_container().set_property("nifi.c2.rest.url.ack", f"https://minifi-c2-server-{context.scenario_id}:10090/c2/config/acknowledge") - context.get_or_create_default_minifi_container().set_property("nifi.c2.flow.base.url", f"https://minifi-c2-server-{context.scenario_id}:10090/c2/config/") - context.get_or_create_default_minifi_container().set_property("nifi.c2.root.classes", "DeviceInfoNode,AgentInformation,FlowInformation,AssetInformation") + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.rest.url", + f"https://minifi-c2-server-{context.scenario_id}:10090/c2/config/heartbeat", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.rest.url.ack", + f"https://minifi-c2-server-{context.scenario_id}:10090/c2/config/acknowledge", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.flow.base.url", + f"https://minifi-c2-server-{context.scenario_id}:10090/c2/config/", + ) + context.get_or_create_default_minifi_container().set_property( + "nifi.c2.root.classes", + "DeviceInfoNode,AgentInformation,FlowInformation,AssetInformation", + ) context.get_or_create_default_minifi_container().set_property("nifi.c2.full.heartbeat", "false") context.get_or_create_default_minifi_container().set_property("nifi.c2.agent.class", "minifi-test-class") context.get_or_create_default_minifi_container().set_property("nifi.c2.agent.identifier", "minifi-test-id") @@ -99,9 +126,12 @@ def start_minifi_c2_server(context: MinifiTestContext): assert context.containers["minifi-c2-server"].deploy(context) -@then("the MiNiFi C2 server logs contain the following message: \"{log_message}\" in less than {duration}") +@then('the MiNiFi C2 server logs contain the following message: "{log_message}" in less than {duration}') def verify_c2_server_logs_contain_message(context: MinifiTestContext, log_message: str, duration: str): duration_seconds = humanfriendly.parse_timespan(duration) - assert wait_for_condition(condition=lambda: log_message in context.containers["minifi-c2-server"].get_logs(), - timeout_seconds=duration_seconds, bail_condition=lambda: context.containers["minifi-c2-server"].exited, - context=context) + assert wait_for_condition( + condition=lambda: log_message in context.containers["minifi-c2-server"].get_logs(), + timeout_seconds=duration_seconds, + bail_condition=lambda: context.containers["minifi-c2-server"].exited, + context=context, + ) diff --git a/minifi_rust/extensions/minifi_rs_playground/features/environment.py b/minifi_rust/extensions/minifi_rs_playground/features/environment.py index 3ba4288e03..8be40cabcd 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/environment.py +++ b/minifi_rust/extensions/minifi_rs_playground/features/environment.py @@ -16,24 +16,22 @@ # under the License. import os -from typing import List from minifi_behave.containers.docker_image_builder import DockerImageBuilder -from minifi_behave.core.hooks import common_after_scenario -from minifi_behave.core.hooks import common_before_scenario, get_minifi_container_image +from minifi_behave.core.hooks import ( + common_after_scenario, + common_before_scenario, + get_minifi_container_image, +) from minifi_behave.core.minifi_test_context import MinifiTestContext -def add_extension_to_minifi_container( - extension_name: str, possible_paths: List[str], context: MinifiTestContext -): +def add_extension_to_minifi_container(extension_name: str, possible_paths: list[str], context: MinifiTestContext): new_container_name = f"apacheminificpp:{extension_name}" is_windows = os.name == "nt" if is_windows: lib_filename = f"{extension_name}.dll" - container_extension_dir = ( - "C:/Program Files/ApacheNiFiMiNiFi/nifi-minifi-cpp/extensions" - ) + container_extension_dir = "C:/Program Files/ApacheNiFiMiNiFi/nifi-minifi-cpp/extensions" else: lib_filename = f"lib{extension_name}.so" container_extension_dir = "/opt/minifi/minifi-current/extensions/" @@ -44,9 +42,7 @@ def add_extension_to_minifi_container( host_path = os.path.join(path, lib_filename) break - assert host_path is not None, ( - f"Could not find {lib_filename} in {[p for p in possible_paths]}" - ) + assert host_path is not None, f"Could not find {lib_filename} in {[p for p in possible_paths]}" with open(host_path, "rb") as f: lib_content = f.read() diff --git a/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py b/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py index f9a0ed64a5..26301f0c29 100644 --- a/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py +++ b/minifi_rust/extensions/minifi_rs_playground/features/steps/steps.py @@ -15,15 +15,16 @@ # specific language governing permissions and limitations # under the License. -from behave import then, when, given import humanfriendly - -from minifi_behave.steps import checking_steps # noqa: F401 -from minifi_behave.steps import configuration_steps # noqa: F401 -from minifi_behave.steps import core_steps # noqa: F401 -from minifi_behave.steps import flow_building_steps # noqa: F401 +from behave import given, then, when from minifi_behave.core.helpers import wait_for_condition from minifi_behave.core.minifi_test_context import MinifiTestContext +from minifi_behave.steps import ( + checking_steps, # noqa: F401 + configuration_steps, # noqa: F401 + core_steps, # noqa: F401 + flow_building_steps, # noqa: F401 +) @when("the MiNiFi instance is started without assertions") @@ -54,9 +55,5 @@ def minifi_logs_processor_metrics(context: MinifiTestContext): context.get_or_create_default_minifi_container().set_property( "nifi.metrics.publisher.LogMetricsPublisher.logging.interval", "1s" ) - context.get_or_create_default_minifi_container().set_property( - "nifi.metrics.publisher.class", "LogMetricsPublisher" - ) - context.get_or_create_default_minifi_container().set_property( - "nifi.metrics.publisher.agent.identifier", "Agent1" - ) + context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.class", "LogMetricsPublisher") + context.get_or_create_default_minifi_container().set_property("nifi.metrics.publisher.agent.identifier", "Agent1") diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000000..f0c1f8ff88 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,17 @@ +line-length = 120 +builtins = ["log", "REL_SUCCESS", "REL_FAILURE", "REL_ORIGINAL", "raw_input"] + +extend-exclude = [ + "thirdparty", + "build*", + "cmake-build-*", +] + +[lint] +# BLE001 (blind-except) is intentional at framework boundaries +ignore = ["BLE001"] + +[lint.per-file-ignores] +# G010 (logging-warn) is intentional in our Python code, because NiFi logging syntax differs +"extensions/python/*" = ["G010"] +"examples/scripts/python/*" = ["G010"] diff --git a/run_flake8.sh b/run_flake8.sh deleted file mode 100755 index 2288893bc5..0000000000 --- a/run_flake8.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/bin/bash -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -set -euo pipefail - -directory=${1:-.} -flake8 --exclude .venv,venv,thirdparty,build,cmake-build-*,github_env --builtins log,REL_SUCCESS,REL_FAILURE,REL_ORIGINAL,raw_input --ignore E501,W503,F811 "${directory}" diff --git a/thirdparty/google-styleguide/cpplint.py b/thirdparty/google-styleguide/cpplint.py index 60ced35354..b0fc73ce77 100644 --- a/thirdparty/google-styleguide/cpplint.py +++ b/thirdparty/google-styleguide/cpplint.py @@ -1699,6 +1699,7 @@ class _IncludeError(Exception): pass + class FileInfo: """Provides utility functions for filenames. @@ -3192,12 +3193,14 @@ class _WrappedInfo(_BlockInfo): pass + class _MemInitListInfo(_WrappedInfo): """Stores information about member initializer lists.""" pass + class _PreprocessorInfo: """Stores checkpoints of nesting stacks when #if/#else is seen.""" diff --git a/thirdparty/rocksdb/all/conanfile.py b/thirdparty/rocksdb/all/conanfile.py index 349f79b9ec..69d640643d 100644 --- a/thirdparty/rocksdb/all/conanfile.py +++ b/thirdparty/rocksdb/all/conanfile.py @@ -16,6 +16,7 @@ import os import glob import shutil +from typing import ClassVar from conan import ConanFile from conan.errors import ConanInvalidConfiguration @@ -37,7 +38,7 @@ class RocksDBConan(ConanFile): topics = ("database", "leveldb", "facebook", "key-value") package_type = "library" settings = "os", "arch", "compiler", "build_type" - options = { + options: ClassVar[dict] = { "shared": [True, False], "fPIC": [True, False], "lite": [True, False], @@ -52,7 +53,7 @@ class RocksDBConan(ConanFile): "enable_sse": [False, "sse42", "avx2"], "use_rtti": [True, False], } - default_options = { + default_options: ClassVar[dict] = { "shared": False, "fPIC": True, "lite": False,