diff --git a/packages/ns-api/README.md b/packages/ns-api/README.md index 0279e275e..5089aba21 100644 --- a/packages/ns-api/README.md +++ b/packages/ns-api/README.md @@ -3570,7 +3570,10 @@ Data can be limited and paginated by using the `limit` and `page` parameters: api-cli ns.dpi list-applications --data '{"limit": 10, "page": 3}' ``` -**PLEASE NOTE**: `category` field can be missing in some applications/protocols. +**PLEASE NOTE**: + +- `category` field can be missing in some applications/protocols. +- applications and protocols are the ones loaded in memory by the netifyd engine; if the engine can't be queried, applications are read from the installed signatures file. Example response: ```json @@ -3628,255 +3631,350 @@ Example response: } ``` -### list-popular +### list-loaded-applications -List popular applications and protocols: +List the applications the netifyd engine has loaded in memory, as returned by `netifyd --dump-apps`: ```bash -api-cli ns.dpi list-popular +api-cli ns.dpi list-loaded-applications ``` -Data can be limited and paginated by using the `limit` and `page` parameters: +The engine loads only the applications available inside the installed signatures, so the response can be matched +against `list-application-catalog` to find out which applications of the catalog are usable on the machine. -```bash -api-cli ns.dpi list-popular --data '{"limit": 3, "page": 2}' +Example response: +```json +{ + "values": [ + { + "id": 133, + "name": "netify.netflix" + }, + { + "id": 10119, + "name": "netify.adobe" + }, + { + "id": 10552, + "name": "netify.tesla" + } + ] +} ``` -**PLEASE NOTE**: +If the engine can't be queried, the response is: +```json +{ + "error": "applications_not_available" +} +``` -- `category` field can be missing in some applications/protocols. -- `missing` field is true when the application/protocol is not available in the current netifyd database. +### list-loaded-protocols + +List the protocols the netifyd engine has loaded in memory, as returned by `netifyd --dump-protos`: + +```bash +api-cli ns.dpi list-loaded-protocols +``` + +Like `list-loaded-applications`, the response can be matched against `list-protocol-catalog` to find out which +protocols of the catalog are usable on the machine. The engine can also report protocols that are not part of the +catalog, like the `Unknown` pseudo-protocol with id `0`. Example response: +```json +{ + "values": [ + { + "id": 116, + "name": "Warcraft3" + }, + { + "id": 121, + "name": "Dropbox" + }, + { + "id": 130, + "name": "HTTP/Connect" + } + ] +} +``` +If the engine can't be queried, the response is: +```json +{ + "error": "protocols_not_available" +} +``` + +### list-appgroups + +List the application groups, ordered by name: + +```bash +api-cli ns.dpi list-appgroups +``` + +Data can be limited and paginated with the `limit` and `page` parameters, and filtered by name with +`search`: + +```bash +api-cli ns.dpi list-appgroups --data '{"search": "business", "limit": 10, "page": 2}' +``` + +Without `limit` every group is returned, which is what the rule drawer needs to fill its group selector. + +Example response: ```json { "values": { "data": [ { - "id": 10392, - "name": "netify.apple-siri", - "type": "application", - "category": { - "name": "business" - }, - "missing": false - }, - { - "id": 142, - "name": "WhatsApp", - "type": "protocol", - "category": { - "name": "messaging" - }, - "missing": false - }, - { - "name": "whatsapp", - "missing": true - }, - { - "id": 238, - "name": "Apple/Push", - "type": "protocol", - "missing": false + "id": "ns_1a2b3c4d5", + "name": "Business and technology services", + "applications": ["netify.amazon", "netify.apple"], + "application_categories": ["cybersecurity"], + "protocols": ["HTTP/Connect"], + "protocol_categories": ["games"], + "used": true, + "matches": ["dpi/ns_9f8e7d6c5"] } ], "meta": { - "last_page": 3, - "total": 8 + "last_page": 1, + "total": 1 } } } ``` +`used` and `matches` report the rules referencing the group: a group in use can't be deleted. -### list-rules +### add-appgroup -List created rules: +Create an application group: ```bash -api-cli ns.dpi list-rules +api-cli ns.dpi add-appgroup --data '{ + "name": "Business and technology services", + "applications": ["netify.amazon"], + "application_categories": ["cybersecurity"], + "protocols": ["HTTP/Connect"], + "protocol_categories": ["games"] +}' ``` -Example response: +- `name`: mandatory, up to 64 characters, unique among the groups regardless of case +- the four member lists are all optional, but the group must hold at least one member overall +- applications and protocols are named as the **DPI engine** reports them, i.e. the `name` field of + `list-loaded-applications` / `list-loaded-protocols`, **not** the `tag` of the catalogs. For + applications the two coincide (`netify.amazon`); for protocols they don't (`HTTP/Connect` in the + engine, `http-connect` in the catalog) and only the engine name can be matched +- categories are named by tag (`cybersecurity`, `games`), which is the same in the engine and in the + catalogs. Application and protocol categories are distinct vocabularies +Members are validated against what the engine has loaded, with the catalog accepted as well for +applications, so that a group stays editable on a machine whose premium signatures were removed. When +neither source can be read the values are stored unchecked: an unknown value never matches, so refusing +the write would be worse than accepting it. Values are always rejected when empty or when they hold a +character that could break the generated expression. + +Example response: ```json { - "values": [ - { - "config-name": "ns_3869dc35", - "enabled": true, - "device": "eth4", - "interface": "GREEN_1", - "action": "block", - "criteria": [ - { - "id": 156, - "name": "netify.spotify", - "type": "application", - "category": { - "name": "streaming-media" - } - }, - { - "id": 10119, - "name": "netify.adobe", - "type": "application", - "category": { - "name": "business" - } - } - ] - }, - { - "config-name": "ns_f1c6e9e0", - "enabled": false, - "interface": "eth4", - "action": "block", - "criteria": [ - { - "id": 196, - "name": "HTTP/S", - "type": "protocol", - "category": { - "name": "web" - } - } - ] - } - ] + "id": "ns_1a2b3c4d5" } ``` -### add-rule +### edit-appgroup -Add DPI rule: +Same payload as `add-appgroup`, plus the `id` of the group. Lists left out are emptied: ```bash -api-cli ns.dpi add-rule --data '{"enabled": false, "device": "eth4", "applications": [], "protocols": ["HTTP/S"]}' +api-cli ns.dpi edit-appgroup --data '{ + "id": "ns_1a2b3c4d5", + "name": "Business", + "applications": ["netify.amazon", "netify.apple"] +}' ``` -Rundown of required parameters: +Example response: +```json +{ + "id": "ns_1a2b3c4d5" +} +``` -- `enabled`: `true` or `false` -- `device`: device name, e.g. `eth4` -- `applications`: list of application names, e.g. `["netify.spotify", "netify.adobe"]`, refer to `list-applications` - api. -- `protocols`: list of protocol names, e.g. `["HTTP/S"]`, refer to `list-applications` api. +### delete-appgroup -Example response: +Delete an application group: + +```bash +api-cli ns.dpi delete-appgroup --data '{"id": "ns_1a2b3c4d5"}' +``` + +Deleting a group referenced by a rule is refused with the list of the referencing rules: +```json +{ + "validation": { + "errors": [ + { + "parameter": "id", + "message": "appgroup_is_used", + "value": ["dpi/ns_9f8e7d6c5"] + } + ] + } +} +``` +Example response: ```json { "message": "success" } ``` -### delete-rule +### list-rules -Delete DPI rule: +List every rule in priority order, which is the order they are evaluated in: the first rule matching a +flow wins and stops the evaluation. ```bash -api-cli nd.dpi delete-rule --data '{"config-name": "ns_f1c6e9e0"}' +api-cli ns.dpi list-rules ``` -Required parameters: - -- `config-name`: rule name, refer to `list-rules` api. +The list is never paginated nor filtered: it is the whole set, which is also what `order-rules` needs. +Rules hidden with `ns_visible '0'` are the only ones left out. `managed` tells whether the rule was +created through the API: an unmanaged rule can be renamed, enabled, reordered and deleted, but not +edited, and it carries the raw `criteria` it matches on. Example response: ```json { - "message": "success" + "values": [ + { + "id": "ns_3869dc35", + "name": "Block streaming", + "enabled": true, + "action": "block", + "source": ["192.168.1.0/24"], + "appgroups": [ + { + "id": "ns_1a2b3c4d5", + "name": "Streaming services" + } + ], + "managed": true, + "index": 0 + }, + { + "id": "ns_f1c6e9e0", + "name": "Legacy rule", + "enabled": true, + "action": "block", + "source": [], + "appgroups": [], + "managed": false, + "criteria": "local_ip == 192.168.100.22 && app == 'netify.facebook';", + "index": 1 + } + ] } ``` -### edit-rule +### add-rule -Edit DPI rule: +Add a DPI rule: ```bash -api-cli ns.dpi edit-rule --data '{"config-name": "ns_f1c6e9e0", "enabled": true, "device": "eth4", "applications": ["netify.spotify", "netify.adobe"], "protocols": []}' +api-cli ns.dpi add-rule --data '{ + "name": "Block streaming", + "enabled": true, + "action": "block", + "source": ["192.168.1.0/24"], + "appgroups": ["ns_1a2b3c4d5"], + "position": "bottom" +}' ``` -Rundown of required parameters: -- `config-name`: rule name, refer to `list-rules` api. +Parameters: + +- `name`: mandatory, up to 64 characters - `enabled`: `true` or `false` -- `device`: device name, e.g. `eth4` -- `applications`: list of application names, e.g. `["netify.spotify", "netify.adobe"]`, refer to `list-applications` - api. -- `protocols`: list of protocol names, e.g. `["HTTP/S"]`, refer to `list-applications` api. +- `action`: `block` or `allow`. `allow` blocks nothing: it labels the flow and, since every rule halts + on match, stops the evaluation before any block rule below it can fire +- `source`: list of addresses, networks or ranges, IPv4 and IPv6 alike, e.g. + `["192.168.1.1", "192.168.1.0/24", "10.0.0.10-10.0.0.20"]`. An empty list matches every host. Ranges + are expanded to CIDR blocks when the rule is generated +- `appgroups`: config names of the application groups the rule matches, refer to `list-appgroups`. **At + least one is required**: a rule with a source and no group would be an IP-level rule, which the + firewall does better, and a rule with neither would match nothing +- `position`: `top` to evaluate the rule before every other one, `bottom` (the default) after them + +There is no device or interface parameter: per-interface rules are not reachable from the API. Example response: ```json { - "message": "success" + "id": "ns_3869dc35" } ``` -### list-devices +### edit-rule -List available devices to be added to DPI rules: +Same payload as `add-rule` minus `position`, plus the `id` of the rule. The priority is left untouched, +use `order-rules` to move the rule. ```bash -api-cli ns.dpi list-devices +api-cli ns.dpi edit-rule --data '{ + "id": "ns_3869dc35", + "name": "Block streaming", + "enabled": false, + "action": "block", + "source": [], + "appgroups": ["ns_1a2b3c4d5"] +}' ``` +Only managed rules can be edited: editing a rule carrying a hand-written criteria is refused with +`rule_not_managed`, because it has no source and no group to rebuild it from. + Example response: ```json { - "values": [ - { - "interface": "GREEN_1", - "device": "eth0" - }, - { - "interface": "GREEN_2", - "device": "eth4" - } - ] + "id": "ns_3869dc35" } ``` -### list-exemptions +### delete-rule -List configured global exemptions: +Delete a DPI rule. The priorities of the remaining rules are renumbered, so no gap is left behind: ```bash -api-cli ns.dpi list-exemptions +api-cli ns.dpi delete-rule --data '{"id": "ns_f1c6e9e0"}' ``` Example response: + ```json { - "values": [ - { - "config-name": "cfg024ffe", - "enabled": true, - "criteria": "192.168.1.1", - "description": "my ex" - } - ] + "message": "success" } ``` -### add-exemption -Add global exemption: +### rename-rule + +Rename a rule, managed or not: ```bash -api-cli ns.dpi add-rule --data '{"criteria": "192.168.1.1", "description": "my host", "enabled": true}' +api-cli ns.dpi rename-rule --data '{"id": "ns_f1c6e9e0", "name": "Legacy rule"}' ``` -Rundown of required parameters: - -- `enabled`: `true` or `false` -- `critera`: an IP address like `192.168.1.1` -- `description`: an optional description - Example response: ```json @@ -3885,34 +3983,30 @@ Example response: } ``` -It can raise a validation error if the criteria is duplicated. Example: +### enable-rule + +Enable a rule, managed or not: + +```bash +api-cli ns.dpi enable-rule --data '{"id": "ns_f1c6e9e0"}' +``` + +Example response: ```json { - "validation": { - "errors": [ - { - "parameter": "criteria", - "message": "criteria_already_exists", - "value": "192.168.1.3" - } - ] - } + "message": "success" } ``` -### delete-exemption +### disable-rule -Delete global exemption rule: +Disable a rule, managed or not: ```bash -api-cli nd.dpi delete-exemption --data '{"config-name": "ns_f1c6e9e0"}' +api-cli ns.dpi disable-rule --data '{"id": "ns_f1c6e9e0"}' ``` -Required parameters: - -- `config-name`: exemption name, refer to `list-exemptions` api. - Example response: ```json @@ -3921,25 +4015,23 @@ Example response: } ``` -### edit-rule +### order-rules -Edit global exemption: +Reorder the rules, renumbering their priorities from the given order: ```bash -api-cli ns.dpi edit-rule --data '{"config-name": "ns_f1c6e9e0", "criteria": "192.168.1.1", "description": "my host", "enabled": true}' +api-cli ns.dpi order-rules --data '{"order": ["ns_f1c6e9e0", "ns_3869dc35"]}' ``` -Rundown of required parameters: -- `config-name`: rule name, refer to `list-rules` api. -- `enabled`: `true` or `false` -- `critera`: an IP address like `192.168.1.1` -- `description`: an optional description +The order must name **every** rule exactly once, hidden ones included, as `list-rules` returns them. A +partial order is refused with `invalid_order`: silently moving the rules left out is never what the +caller meant. Example response: ```json { - "message": "success" + "values": ["ns_f1c6e9e0", "ns_3869dc35"] } ``` diff --git a/packages/ns-api/files/ns.dpi b/packages/ns-api/files/ns.dpi index 84c3dadc5..cac3b9202 100755 --- a/packages/ns-api/files/ns.dpi +++ b/packages/ns-api/files/ns.dpi @@ -1,7 +1,7 @@ #!/usr/bin/python3 # -# Copyright (C) 2023 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # @@ -22,45 +22,66 @@ if cmd == 'list': }, 'list-rules': {}, 'add-rule': { + 'name': 'str', 'enabled': False, - 'device': 'str', - 'applications': 'str', - 'protocols': 'str' - }, - 'delete-rule': { - 'config-name': 'str' + 'action': 'str', + 'source': [], + 'appgroups': [], + 'position': 'str' }, 'edit-rule': { - 'config-name': 'str', + 'id': 'str', + 'name': 'str', 'enabled': False, - 'device': 'str', - 'applications': 'str', - 'protocols': 'str' + 'action': 'str', + 'source': [], + 'appgroups': [] }, - 'add-exemption': { - 'criteria': 'str', - 'description': 'str', - 'enabled': False + 'delete-rule': { + 'id': 'str' }, - 'delete-exemption': { - 'config-name': 'str' + 'rename-rule': { + 'id': 'str', + 'name': 'str' }, - 'edit-exemption': { - 'config-name': 'str', - 'criteria': 'str', - 'description': 'str', - 'enabled': False + 'enable-rule': { + 'id': 'str' }, - 'list-exemptions': {}, - 'list-devices': {}, - 'list-popular': { - 'limit': 32, - 'page': 32 + 'disable-rule': { + 'id': 'str' + }, + 'order-rules': { + 'order': ['ns_1234', 'ns_5678'] }, 'list-application-categories': {}, 'list-application-catalog': {}, + 'list-loaded-applications': {}, 'list-protocol-categories': {}, - 'list-protocol-catalog': {} + 'list-protocol-catalog': {}, + 'list-loaded-protocols': {}, + 'list-appgroups': { + 'search': 'str', + 'limit': 32, + 'page': 32 + }, + 'add-appgroup': { + 'name': 'str', + 'applications': [], + 'application_categories': [], + 'protocols': [], + 'protocol_categories': [] + }, + 'edit-appgroup': { + 'id': 'str', + 'name': 'str', + 'applications': [], + 'application_categories': [], + 'protocols': [], + 'protocol_categories': [] + }, + 'delete-appgroup': { + 'id': 'str' + } })) elif cmd == 'call': action = sys.argv[2] @@ -80,6 +101,14 @@ elif cmd == 'call': print(json.dumps({'values': content})) except Exception: print(json.dumps({})) + elif action == 'list-loaded-applications': + try: + applications = dpi.load_applications() + except Exception: + print(json.dumps(utils.generic_error('applications_not_available'))) + else: + print(json.dumps({'values': [{'id': app_id, 'name': app_name} + for app_id, app_name in applications.items()]})) elif action == 'list-protocol-categories': try: with open('/etc/netifyd/netify-protocol-categories.json', 'r') as f: @@ -94,6 +123,14 @@ elif cmd == 'call': print(json.dumps({'values': content})) except Exception: print(json.dumps({})) + elif action == 'list-loaded-protocols': + try: + protocols = dpi.load_protocols() + except Exception: + print(json.dumps(utils.generic_error('protocols_not_available'))) + else: + print(json.dumps({'values': [{'id': proto_id, 'name': proto_name} + for proto_id, proto_name in protocols.items()]})) elif action == 'list-applications': data = json.JSONDecoder().decode(sys.stdin.read()) result = dpi.list_applications(data.get('search', None), data.get('limit', None), data.get('page', 1)) @@ -102,37 +139,55 @@ elif cmd == 'call': print(json.dumps({'values': dpi.list_rules(e_uci)})) elif action == 'add-rule': data = json.JSONDecoder().decode(sys.stdin.read()) - dpi.add_rule(e_uci, data['enabled'], data['device'], 'block', data['applications'], data['protocols']) - print(json.dumps({'message': 'success'})) + config_name = dpi.add_rule(e_uci, data['name'], data['enabled'], data['action'], + data.get('source', []), data.get('appgroups', []), + data.get('position', 'bottom')) + print(json.dumps({'id': config_name})) + elif action == 'edit-rule': + data = json.JSONDecoder().decode(sys.stdin.read()) + config_name = dpi.edit_rule(e_uci, data['id'], data['name'], data['enabled'], data['action'], + data.get('source', []), data.get('appgroups', [])) + print(json.dumps({'id': config_name})) elif action == 'delete-rule': data = json.JSONDecoder().decode(sys.stdin.read()) - dpi.delete_rule(e_uci, data['config-name']) + dpi.delete_rule(e_uci, data['id']) print(json.dumps({'message': 'success'})) - elif action == 'edit-rule': + elif action == 'rename-rule': data = json.JSONDecoder().decode(sys.stdin.read()) - dpi.edit_rule(e_uci, data['config-name'], data['enabled'], data['device'], 'block', data['applications'], - data['protocols']) + dpi.rename_rule(e_uci, data['id'], data['name']) print(json.dumps({'message': 'success'})) - elif action == 'add-exemption': + elif action == 'enable-rule': data = json.JSONDecoder().decode(sys.stdin.read()) - dpi.add_exemption(e_uci, data['criteria'], data['description'], data['enabled']) + dpi.enable_rule(e_uci, data['id']) print(json.dumps({'message': 'success'})) - elif action == 'delete-exemption': + elif action == 'disable-rule': data = json.JSONDecoder().decode(sys.stdin.read()) - dpi.delete_exemption(e_uci, data['config-name']) + dpi.disable_rule(e_uci, data['id']) print(json.dumps({'message': 'success'})) - elif action == 'edit-exemption': + elif action == 'order-rules': data = json.JSONDecoder().decode(sys.stdin.read()) - dpi.edit_exemption(e_uci, data['config-name'], data['criteria'], data['description'], data['enabled']) - print(json.dumps({'message': 'success'})) - elif action == 'list-exemptions': - print(json.dumps({'values': dpi.list_exemptions(e_uci)})) - elif action == 'list-devices': - print(json.dumps({'values': dpi.list_devices(e_uci)})) - elif action == 'list-popular': + print(json.dumps({'values': dpi.order_rules(e_uci, data['order'])})) + elif action == 'list-appgroups': data = json.JSONDecoder().decode(sys.stdin.read()) - result = dpi.list_popular(e_uci, data.get('limit', None), data.get('page', 1)) + result = dpi.list_appgroups(e_uci, data.get('search', None), data.get('limit', None), + data.get('page', 1)) print(json.dumps({'values': result})) + elif action == 'add-appgroup': + data = json.JSONDecoder().decode(sys.stdin.read()) + config_name = dpi.add_appgroup(e_uci, data['name'], data.get('applications', []), + data.get('application_categories', []), data.get('protocols', []), + data.get('protocol_categories', [])) + print(json.dumps({'id': config_name})) + elif action == 'edit-appgroup': + data = json.JSONDecoder().decode(sys.stdin.read()) + config_name = dpi.edit_appgroup(e_uci, data['id'], data['name'], data.get('applications', []), + data.get('application_categories', []), data.get('protocols', []), + data.get('protocol_categories', [])) + print(json.dumps({'id': config_name})) + elif action == 'delete-appgroup': + data = json.JSONDecoder().decode(sys.stdin.read()) + dpi.delete_appgroup(e_uci, data['id']) + print(json.dumps({'message': 'success'})) except KeyError as e: print(json.dumps(utils.validation_error(e.args[0], 'required'))) except json.JSONDecodeError: diff --git a/packages/ns-dpi/Makefile b/packages/ns-dpi/Makefile index 81e4bde6a..c0f72ae34 100644 --- a/packages/ns-dpi/Makefile +++ b/packages/ns-dpi/Makefile @@ -78,7 +78,9 @@ define Package/ns-dpi/install $(INSTALL_BIN) ./files/dpi-license-update.py $(1)/usr/sbin/dpi-license-update $(INSTALL_BIN) ./files/dpi-data-update.py $(1)/usr/sbin/dpi-data-update $(INSTALL_BIN) ./files/dpi-update.py $(1)/usr/sbin/dpi-update + $(INSTALL_BIN) ./files/dpi-migrate.py $(1)/usr/sbin/dpi-migrate $(INSTALL_CONF) ./files/20_dpi $(1)/etc/uci-defaults + $(INSTALL_BIN) ./files/21_dpi_migrate $(1)/etc/uci-defaults/21_dpi_migrate $(INSTALL_BIN) ./files/99-dpi-license-update-cron.uci-defaults $(1)/etc/uci-defaults/99-dpi-license-update-cron $(INSTALL_BIN) ./files/99-dpi-data-update-cron.uci-defaults $(1)/etc/uci-defaults/99-dpi-data-update-cron $(INSTALL_DIR) $(1)/usr/share/ns-plug/hooks/register diff --git a/packages/ns-dpi/files/20_dpi b/packages/ns-dpi/files/20_dpi index f5e6adaf2..da9094280 100644 --- a/packages/ns-dpi/files/20_dpi +++ b/packages/ns-dpi/files/20_dpi @@ -6,24 +6,6 @@ config main 'config' option log_blocked '0' option enabled '0' option firewall_exemption '0' - list popular_filters 'netify.facebook' - list popular_filters 'netify.amazon-prime' - list popular_filters 'netify.whatsapp' - list popular_filters 'netify.instagram' - list popular_filters 'netify.netflix' - list popular_filters 'netify.telegram' - list popular_filters 'netify.tiktok' - list popular_filters 'netify.youtube' - list popular_filters 'netify.facebook-messenger' - list popular_filters 'netify.twitter' - list popular_filters 'netify.vimeo' - list popular_filters 'netify.snapchat' - list popular_filters 'netify.pinterest' - list popular_filters 'netify.nordvpn' - list popular_filters 'DoT' - list popular_filters 'netify.twitch' - list popular_filters 'netify.teamviewer' - list popular_filters 'DoH' EOI fi diff --git a/packages/ns-dpi/files/21_dpi_migrate b/packages/ns-dpi/files/21_dpi_migrate new file mode 100755 index 000000000..4521e6801 --- /dev/null +++ b/packages/ns-dpi/files/21_dpi_migrate @@ -0,0 +1,13 @@ +#!/bin/sh + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# Migrate /etc/config/dpi from the schema used before application groups existed. Safe to run on every +# boot: dpi-migrate is a no-op once there is nothing left in the old schema. + +set -e + +/usr/sbin/dpi-migrate diff --git a/packages/ns-dpi/files/connlabel.conf b/packages/ns-dpi/files/connlabel.conf index 784b84ec4..6131ac74c 100644 --- a/packages/ns-dpi/files/connlabel.conf +++ b/packages/ns-dpi/files/connlabel.conf @@ -4,4 +4,5 @@ 3 bulk 4 best_effort 5 video -6 voice \ No newline at end of file +6 voice +7 netify-allowed \ No newline at end of file diff --git a/packages/ns-dpi/files/dpi-config b/packages/ns-dpi/files/dpi-config index 969796d0a..1c114cd08 100755 --- a/packages/ns-dpi/files/dpi-config +++ b/packages/ns-dpi/files/dpi-config @@ -1,7 +1,7 @@ #!/usr/bin/python3 # -# Copyright (C) 2025 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # @@ -9,7 +9,8 @@ from euci import EUci import json import subprocess import os -from nethsec import utils, objects +import syslog +from nethsec import utils, objects, dpi def get_interface_ips(interface): ret = list() @@ -50,6 +51,13 @@ config = { "path": "/var/run/netifyd/", "prefix": "dpi-actions", }, + "allow": { + "target_type": "ctlabel", + "target_enabled": True, + "labels": [ + "netify-allowed" + ] + }, "analyzed": { "target_type": "ctlabel", "target_enabled": True, @@ -58,7 +66,18 @@ config = { ] } }, - "actions": {}, + # analyzed comes first and lets the evaluation continue, so the label is never lost to a rule + # that halts on match + "actions": { + "analyzed": { + "enabled": True, + "priority": 0, + "halt_on_match": False, + "criteria": 'detection_guessed || detection_complete;', + "targets": ["analyzed"], + "exemptions": [] + } + }, "exemptions": [] } @@ -86,77 +105,86 @@ for ex_name in exemptions: rcount = 0 valid_actions = ['block', 'bulk', 'best_effort', 'video', 'voice'] +legacy_fields = ('device', 'application', 'protocol', 'category') for section in u.get_all('dpi'): if u.get('dpi', section) != 'rule': continue rule = u.get_all('dpi', section) - if rule['action'] not in valid_actions or not rule['enabled']: + if not rule.get('enabled'): continue - device = rule.get('device', '*') - - if 'criteria' in rule: - # criteria has precedence over sources, protocol, category and application - criteria = rule['criteria'].replace('"',"'") - else: - # generate criteria from source, protocol, category and application - sources = [] - for source in rule.get('source', []): - if objects.is_object_id(source): - for ip in objects.get_object_ips(u, source): - sources.append(f'local_ip == {ip}') - else: # custom local IPs - sources.append(f'local_ip == {source}') - - applications = [] - for app in rule.get('application', []): - applications.append(f"app == '{app}'") - for proto in rule.get('protocol', []): - applications.append(f"proto == '{proto.lower()}'") - for cat in rule.get('category', []): - applications.append(f"category == '{cat.lower()}'") - - sources_s = ' or '.join(sources) - applications_s = ' or '.join(applications) - criteria = f'(iface_nfq_src == \'{device}\' or iface_nfq_dst == \'{device}\') && ' - if len(sources) < 1: - criteria += f'({applications_s}) ;' - elif len(applications) < 1: - criteria += f'({sources_s}) ;' - else: - criteria += f'({sources_s}) && ({applications_s}) ;' + # a rule from the schema used before application groups existed carries at least one of these + # fields; their presence is a permanent marker, unrelated to whether the migration has since given + # the section a priority + is_legacy = any(field in rule for field in legacy_fields) + + if not is_legacy and 'criteria' not in rule: + # current schema: application groups, optionally narrowed to a source, or a rule migrated from + # a global exemption (source only, no group — dpi.migrate_schema) + if rule.get('action') not in dpi.DPI_RULE_ACTIONS: + syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, unknown action {rule.get('action')}") + continue + criteria = dpi.build_rule_criteria(u, rule, require_appgroup=False) + if not criteria: + # never emit an empty criteria: it is a parse error, and a parse error kills the agent + syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, it matches nothing") + continue + try: + priority = int(rule.get('priority', 1)) + except ValueError: + syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, invalid priority") + continue + config["actions"][section] = { + "enabled": rule['enabled'] == '1', + "priority": priority, + "halt_on_match": True, + "criteria": criteria, + "targets": [rule['action']], + "exemptions": [] + } + continue - vlan_id = None - base_if = None - for item in utils.get_all_by_type(u, 'network', 'device').values(): - if item.get('vid', None) is not None and item.get('name', '') == device: - vlan_id = item.get('vid', None) - base_if = item.get('ifname', None) - break + if not is_legacy: + # raw criteria: hand-written, or a rule migrated from the old schema with its behaviour frozen + # verbatim (dpi.migrate_schema) — halts on match and keeps its own priority, same as any other + # rule of the current schema + if rule.get('action') not in dpi.DPI_RULE_ACTIONS: + syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, unknown action {rule.get('action')}") + continue + try: + priority = int(rule.get('priority', 1)) + except ValueError: + syslog.syslog(syslog.LOG_WARNING, f"dpi-config: skipping {section}, invalid priority") + continue + config["actions"][section] = { + "enabled": rule['enabled'] == '1', + "priority": priority, + "halt_on_match": True, + "criteria": dpi.freeze_legacy_criteria(u, rule), + "targets": [rule['action']], + "exemptions": [] + } + continue - if vlan_id is not None and base_if is not None: - criteria = f'vlan_id == {vlan_id} && {criteria}' - device = base_if + # legacy rule, still on the old schema: emitted as it was until the migration converts it + if rule.get('action') not in valid_actions: + continue targets = [rule['action']] if 'log' in rule and rule['log'] == '1': targets.append('log') config["actions"][f"rule{rcount}"] = { "enabled": rule['enabled'] == '1', - "criteria": criteria, + # legacy rules never halt, so they are evaluated after the ones of the current schema and + # their relative order does not matter + "priority": 1000 + rcount, + "halt_on_match": False, + "criteria": dpi.freeze_legacy_criteria(u, rule), "targets": targets, "exemptions": rule.get('exemption', []) } rcount = rcount + 1 -# add global analyzed for all flows that are not blocked -config["actions"]["analyzed"] = { - "enabled": True, - "criteria": 'detection_guessed || detection_complete;', - "targets": ["analyzed"], - "exemptions": [] -} - # if directories do not exist, create them if not os.path.exists(os.path.dirname(cfg_file)): os.makedirs(os.path.dirname(cfg_file)) diff --git a/packages/ns-dpi/files/dpi-migrate.py b/packages/ns-dpi/files/dpi-migrate.py new file mode 100755 index 000000000..1a3b32d15 --- /dev/null +++ b/packages/ns-dpi/files/dpi-migrate.py @@ -0,0 +1,13 @@ +#!/usr/bin/env python3 + +# +# Copyright (C) 2026 Nethesis S.r.l. +# SPDX-License-Identifier: GPL-2.0-only +# + +# this script is supposed to be run by the 21_dpi_migrate uci-defaults + +from euci import EUci +from nethsec import dpi + +dpi.migrate_schema(EUci()) diff --git a/packages/ns-ui/Makefile b/packages/ns-ui/Makefile index 5403c6c47..84626d0b5 100644 --- a/packages/ns-ui/Makefile +++ b/packages/ns-ui/Makefile @@ -12,7 +12,7 @@ PKG_RELEASE:=1 PKG_SOURCE_PROTO:=git PKG_SOURCE_URL:=https://github.com/NethServer/nethsecurity-ui.git -PKG_SOURCE_VERSION:=$(PKG_VERSION) +PKG_SOURCE_VERSION:=cab1649acd4fc2034d8616a29b4a1fbc4037fa0d PKG_SOURCE_SUBDIR:=nethsecurity-ui-$(PKG_SOURCE_VERSION) PKG_BUILD_DIR:=$(BUILD_DIR)/$(PKG_SOURCE_SUBDIR) PKG_MIRROR_HASH:=skip diff --git a/packages/python3-nethsec/src/nethsec/dpi/__init__.py b/packages/python3-nethsec/src/nethsec/dpi/__init__.py index 5de35668e..57cc50cd5 100644 --- a/packages/python3-nethsec/src/nethsec/dpi/__init__.py +++ b/packages/python3-nethsec/src/nethsec/dpi/__init__.py @@ -1,7 +1,7 @@ #!/usr/bin/python3 # -# Copyright (C) 2023 Nethesis S.r.l. +# Copyright (C) 2026 Nethesis S.r.l. # SPDX-License-Identifier: GPL-2.0-only # @@ -9,14 +9,15 @@ Library that handles the DPI rules. """ +import ipaddress import json import subprocess -from fnmatch import fnmatch +import syslog import math from euci import EUci -from nethsec import utils, firewall +from nethsec import objects, utils from nethsec.utils import ValidationError @@ -36,6 +37,32 @@ def __load_applications() -> dict[int, str]: return applications +def load_applications() -> dict[int, str]: + """ + Reads the applications loaded in memory by the engine, using the netifyd --dump-apps command. + + Returns: + dict of applications, the key is the application id and the value is the application name + """ + result = subprocess.run(['netifyd', '--dump-apps'], check=True, capture_output=True) + applications = dict[int, str]() + for line in result.stdout.decode().splitlines(): + # lines can be empty + if len(line.strip()) < 1: + continue + line_split = line.split(":", 1) + if len(line_split) < 2: + continue + try: + application_id = int(line_split[0].strip()) + except ValueError: + # skip lines not in the "id: name" format, like headers + continue + applications[application_id] = line_split[1].strip() + + return applications + + def __load_application_categories() -> dict[int, dict[str]]: """ Reads the application categories from the netify-categories.json file. @@ -130,7 +157,11 @@ def __load_blocklist() -> list[dict[str]]: list of dicts, each dict contains the property "id", "name", "type" and "category" """ result = list[dict[str]]() - applications = __load_applications() + try: + applications = load_applications() + except Exception: + # the engine can't be queried, fall back to the signatures file + applications = __load_applications() application_categories = __load_application_categories() for application_id, application_name in applications.items(): @@ -159,29 +190,6 @@ def __load_blocklist() -> list[dict[str]]: return result -def list_devices(e_uci: EUci): - """ - List device-interface available for filtering. - - Returns: - list of dicts, each dict contains the property "interface" and "device" - """ - instance_name = list(e_uci.get('netifyd').keys())[0] - devices = e_uci.get('netifyd', instance_name, 'internal_if', default=[], list=True) - for zone in firewall.list_zones(e_uci).values(): - if zone['name'] == 'wan': - continue - network_devices = utils.get_all_devices_by_zone(e_uci, zone['name']) - devices = list(set(list(devices) + network_devices)) - ret = [] - for item in devices: - interface_name = utils.get_interface_from_device(e_uci, item) - ret.append({ - 'interface': interface_name if interface_name is not None else item, - 'device': item - }) - return ret - def list_applications(search: str = None, limit: int = None, page: int = 1) -> dict: """ List applications available for filtering. @@ -221,40 +229,297 @@ def list_applications(search: str = None, limit: int = None, page: int = 1) -> d } -def list_popular(e_uci: EUci, limit: int = None, page: int = 1) -> dict: +def __apply(e_uci: EUci): + """ + Commit the dpi config and reload the dpi service. + + Used by the schema migration only: it runs at boot with nobody around to confirm the pending uci + changes, so it must take effect on its own. Rule and appgroup CRUD go through the normal pending + uci changes instead, applied later by ns.commit like every other section. + """ + e_uci.commit('dpi') + subprocess.run(["/etc/init.d/dpi", "reload"], check=True) + + +def __toggle_engine(e_uci: EUci): + count_enabled = 0 + for section in e_uci.get_all('dpi'): + if e_uci.get('dpi', section, default="") == "rule" and e_uci.get('dpi', section, 'enabled', default="0") == "1": + count_enabled = count_enabled + 1 + + if count_enabled > 0: + e_uci.set('dpi', 'config', 'enabled', '1') + else: + e_uci.set('dpi', 'config', 'enabled', '0') + + +# Application groups: a NethSecurity abstraction, the netifyd plugin has no group primitive. +# A group is a named set of match members, expanded into an inline expression at generation time. + +NETIFYD_DATA_DIR = '/etc/netifyd' + +# UCI list name of every member kind, mapped to the API parameter it is fed by +APPGROUP_MEMBERS = { + 'app': 'applications', + 'app_category': 'application_categories', + 'proto': 'protocols', + 'proto_category': 'protocol_categories' +} + +# validation error message of every member kind +__APPGROUP_MEMBER_ERRORS = { + 'app': 'invalid_application', + 'app_category': 'invalid_application_category', + 'proto': 'invalid_protocol', + 'proto_category': 'invalid_protocol_category' +} + +# characters that would let a member value escape its quoted literal, or terminate the expression +__APPGROUP_FORBIDDEN_CHARS = '\'"\\;()\n\r\t' + +APPGROUP_NAME_MAX_LENGTH = 64 + + +def load_categories() -> dict[str, dict[int, str]]: + """ + Reads the categories loaded in memory by the engine, using the netifyd --dump-categories command. + + The dump reports one category per line, as ": : ". + + Returns: + dict of categories indexed by type, e.g. `{'application': {1: 'adult'}, 'protocol': {2: 'database'}}`. + Ids are the ones the agent uses internally: they do not match the ids of the downloaded catalogs, + so categories must always be crossed by tag. + """ + result = subprocess.run(['netifyd', '--dump-categories'], check=True, capture_output=True) + categories = dict[str, dict[int, str]]() + for line in result.stdout.decode().splitlines(): + line_split = line.split(":", 2) + if len(line_split) < 3: + continue + try: + category_id = int(line_split[0].strip()) + except ValueError: + # skip lines not in the "id: type: tag" format, like headers + continue + category_type = line_split[1].strip() + categories.setdefault(category_type, dict[int, str]())[category_id] = line_split[2].strip() + + return categories + + +def __load_catalog_tags(filename: str) -> set[str]: """ - List popular applications available for filtering. + Reads the tags of a downloaded catalog, one of the `/etc/netifyd/netify-*.json` files. Args: - - limit: limit the number of results - - page: page number + - filename: name of the catalog file inside the netifyd data directory Returns: - list of dicts, each dict contains the property "id", "name", "type" and "category" + set of the tags the catalog lists """ - popular_filters = e_uci.get('dpi', 'config', 'popular_filters', default=[], list=True) - block_list = {block['name']: block for block in __load_blocklist()} - result = [] + with open(f'{NETIFYD_DATA_DIR}/{filename}', 'r') as file: + return {entry['tag'] for entry in json.load(file) if entry.get('tag')} + + +def __appgroup_vocabulary(kind: str) -> set[str] | None: + """ + Values a member of the given kind can take, lowercased. + + The engine is the authority: only what it has loaded can be matched. The downloaded catalog backs + applications up, so that a group stays editable on a machine that lost its premium signatures. + + Args: + - kind: one of the keys of `APPGROUP_MEMBERS` - for popular_filter in popular_filters: - if popular_filter in block_list.keys(): - result.append(block_list[popular_filter] | {'missing': False}) + Returns: + set of accepted values, or None when no source is available and the values cannot be checked + """ + vocabulary = set[str]() + try: + if kind == 'app': + vocabulary |= {value.lower() for value in load_applications().values()} + elif kind == 'proto': + vocabulary |= {value.lower() for value in load_protocols().values()} else: - result.append({ - 'name': popular_filter, - 'missing': True - }) + category_type = 'application' if kind == 'app_category' else 'protocol' + vocabulary |= {value.lower() for value in load_categories().get(category_type, {}).values()} + except Exception: + # the engine can't be queried, fall back to the downloaded catalogs below + pass + + if kind != 'proto': + # protocol names come from the agent itself and have no counterpart in the catalog, where the + # same protocol is listed under a different tag: the catalog would accept unmatchable values + catalogs = { + 'app': 'netify-application-catalog.json', + 'app_category': 'netify-application-categories.json', + 'proto_category': 'netify-protocol-categories.json' + } + try: + vocabulary |= {value.lower() for value in __load_catalog_tags(catalogs[kind])} + except Exception: + pass - total = len(result) + # an unknown value never matches, so a missing vocabulary is a reason to skip the check, not to refuse + return vocabulary if vocabulary else None + + +def __validate_appgroup_members(kind: str, values: list[str]) -> list[str]: + """ + Validate the members of one kind, stripping and deduplicating them. + + Args: + - kind: one of the keys of `APPGROUP_MEMBERS` + - values: list of member values + + Returns: + the sanitized list of members + + Raises: + - ValidationError: if a member is empty, holds a character that would break the expression, or is + unknown to both the engine and the catalog + """ + parameter = APPGROUP_MEMBERS[kind] + message = __APPGROUP_MEMBER_ERRORS[kind] + if not values: + return [] + + vocabulary = __appgroup_vocabulary(kind) + members = [] + seen = set[str]() + for value in values: + if not isinstance(value, str): + raise ValidationError(parameter, message, value) + member = value.strip() + if not member or any(char in member for char in __APPGROUP_FORBIDDEN_CHARS): + raise ValidationError(parameter, message, value) + if vocabulary is not None and member.lower() not in vocabulary: + raise ValidationError(parameter, message, member) + if member.lower() not in seen: + seen.add(member.lower()) + members.append(member) + + return members + + +def __validate_appgroup(e_uci: EUci, name: str, applications: list[str], application_categories: list[str], + protocols: list[str], protocol_categories: list[str], + config_name: str = None) -> tuple[str, dict[str, list[str]]]: + """ + Validate a group before it is stored. + + Args: + - e_uci: euci instance + - name: name of the group + - applications: list of application names + - application_categories: list of application category tags + - protocols: list of protocol names + - protocol_categories: list of protocol category tags + - config_name: config name of the group being edited, excluded from the name uniqueness check + + Returns: + a tuple with the sanitized name and the sanitized members, indexed by UCI list name + + Raises: + - ValidationError: if the name or any member is invalid, or if the group has no members + """ + if not isinstance(name, str) or not name.strip(): + raise ValidationError('name', 'name_required', name) + name = name.strip() + if len(name) > APPGROUP_NAME_MAX_LENGTH: + raise ValidationError('name', 'name_too_long', name) + for section, group in (utils.get_all_by_type(e_uci, 'dpi', 'appgroup') or {}).items(): + if section != config_name and group.get('ns_name', '').strip().lower() == name.lower(): + raise ValidationError('name', 'name_already_exists', name) + + members = { + 'app': __validate_appgroup_members('app', applications), + 'app_category': __validate_appgroup_members('app_category', application_categories), + 'proto': __validate_appgroup_members('proto', protocols), + 'proto_category': __validate_appgroup_members('proto_category', protocol_categories) + } + if not any(members.values()): + # an empty group would expand to an empty criteria, which takes the agent down + raise ValidationError('members', 'appgroup_is_empty', '') + + return name, members + + +def __save_appgroup_data(e_uci: EUci, config_name: str, name: str, members: dict[str, list[str]]): + e_uci.set('dpi', config_name, 'ns_name', name) + for uci_list, values in members.items(): + if values: + e_uci.set('dpi', config_name, uci_list, values) + else: + # an emptied list must be removed, not stored empty + e_uci.delete('dpi', config_name, uci_list) + + +def is_used_appgroup(e_uci: EUci, config_name: str) -> tuple[bool, list[str]]: + """ + Check if an application group is referenced by a rule. + + Args: + - e_uci: euci instance + - config_name: config name of the group + + Returns: + A tuple with: + - True if the group is referenced by at least one rule, False otherwise + - a list of the rule sections referencing it + """ + matches = [] + for section, rule in (utils.get_all_by_type(e_uci, 'dpi', 'rule') or {}).items(): + if config_name in rule.get('appgroup', []): + matches.append(f'dpi/{section}') + return len(matches) > 0, matches + + +def list_appgroups(e_uci: EUci, search: str = None, limit: int = None, page: int = 1, + used_info: bool = True) -> dict: + """ + List the application groups, ordered by name. + + Args: + - e_uci: euci instance + - search: search string, matched against the group name + - limit: limit the number of results, all of them if not given + - page: page number + - used_info: include the used and matches info + + Returns: + dict with the "data" list and the "meta" pagination info + """ + groups = [] + for section, group in (utils.get_all_by_type(e_uci, 'dpi', 'appgroup') or {}).items(): + data_group = {'id': section, 'name': group.get('ns_name', '')} + for uci_list, parameter in APPGROUP_MEMBERS.items(): + data_group[parameter] = list(group.get(uci_list, [])) + groups.append(data_group) + + # pagination needs a deterministic order, the UCI one is not + groups.sort(key=lambda group: group['name'].lower()) + + if search: + search = search.lower() + groups = [group for group in groups if search in group['name'].lower()] + + total = len(groups) if limit is not None: - result = result[limit * (page - 1):limit * page] + groups = groups[limit * (page - 1):limit * page] last_page = math.ceil(total / limit) else: last_page = 1 + if used_info: + # only for the returned page: every check walks the rules + for group in groups: + group['used'], group['matches'] = is_used_appgroup(e_uci, group['id']) + return { - 'data': result, + 'data': groups, 'meta': { 'last_page': last_page, 'total': total, @@ -262,235 +527,702 @@ def list_popular(e_uci: EUci, limit: int = None, page: int = 1) -> dict: } -def list_rules(e_uci: EUci) -> list[dict[str]]: +def add_appgroup(e_uci: EUci, name: str, applications: list[str] = None, + application_categories: list[str] = None, protocols: list[str] = None, + protocol_categories: list[str] = None) -> str: + """ + Store a new application group. + + Args: + - e_uci: euci instance + - name: name of the group + - applications: list of application names, as the engine reports them + - application_categories: list of application category tags + - protocols: list of protocol names, as the engine reports them + - protocol_categories: list of protocol category tags + + Returns: + config name of the group created + + Raises: + - ValidationError: if the name or any member is invalid, or if the group has no members + """ + name, members = __validate_appgroup(e_uci, name, applications, application_categories, protocols, + protocol_categories) + config_name = utils.get_random_id() + e_uci.set('dpi', config_name, 'appgroup') + __save_appgroup_data(e_uci, config_name, name, members) + e_uci.save('dpi') + return config_name + + +def edit_appgroup(e_uci: EUci, config_name: str, name: str, applications: list[str] = None, + application_categories: list[str] = None, protocols: list[str] = None, + protocol_categories: list[str] = None) -> str: + """ + Edit an application group. + + Args: + - e_uci: euci instance + - config_name: config name of the group to edit + - name: name of the group + - applications: list of application names, as the engine reports them + - application_categories: list of application category tags + - protocols: list of protocol names, as the engine reports them + - protocol_categories: list of protocol category tags + + Returns: + config name of the group edited + + Raises: + - ValidationError: if the group does not exist, or if the name or any member is invalid + """ + if e_uci.get('dpi', config_name, default=None) != 'appgroup': + raise ValidationError('id', 'appgroup_does_not_exists', config_name) + + name, members = __validate_appgroup(e_uci, name, applications, application_categories, protocols, + protocol_categories, config_name) + __save_appgroup_data(e_uci, config_name, name, members) + e_uci.save('dpi') + return config_name + + +def delete_appgroup(e_uci: EUci, config_name: str) -> str: """ - Index all rules + Delete an application group. Args: - e_uci: euci instance + - config_name: config name of the group to delete Returns: - list of dicts, each dict contains the property "config-name", "description", "enabled", "interface" and "blocks" - """ - rules = list[dict[str]]() - fetch_rules = utils.get_all_by_type(e_uci, 'dpi', 'rule') - - if not fetch_rules: - return rules - - for rule_name in fetch_rules.keys(): - # skipping rules with criteria, must be custom entries - if e_uci.get('dpi', rule_name, 'criteria', default=None) is None: - # load blocklist of applications and protocols - blocklist = __load_blocklist() - # get content of rule - rule = fetch_rules[rule_name] - # prepare the data to append to rules - data_rule = dict[str]() - data_rule['config-name'] = rule_name - data_rule['enabled'] = rule.get('enabled', '1') == '1' - data_rule['device'] = rule.get('device', '*') - # from device, get the interface - interface = utils.get_interface_from_device(e_uci, data_rule['device']) - if interface is not None: - data_rule['interface'] = interface - data_rule['action'] = rule.get('action') - # get the blocked applications/protocols - data_rule['criteria'] = list[dict[str]]() - - # filter by application - application_blocklist = [item for item in blocklist if item['type'] == 'application'] - for application in rule.get('application', []): - found_app = [item for item in application_blocklist if item['name'] == application] - # there's a possibility of not finding the application due to manual edit of the config - if len(found_app) > 0: - data_rule['criteria'].append(found_app[0]) - - # filter by protocol - protocol_blocklist = [item for item in blocklist if item['type'] == 'protocol'] - for protocol in rule.get('protocol', []): - found_protocol = [item for item in protocol_blocklist if item['name'] == protocol] - # there's a possibility of not finding the protocol due to manual edit of the config - if len(found_protocol) > 0: - data_rule['criteria'].append(found_protocol[0]) - - # append rule - rules.append(data_rule) + config name of the group deleted + + Raises: + - ValidationError: if the group does not exist or is referenced by a rule + """ + if e_uci.get('dpi', config_name, default=None) != 'appgroup': + raise ValidationError('id', 'appgroup_does_not_exists', config_name) + + used, matches = is_used_appgroup(e_uci, config_name) + if used: + raise ValidationError('id', 'appgroup_is_used', matches) + + e_uci.delete('dpi', config_name) + e_uci.save('dpi') + return config_name + + +def expand_appgroups(e_uci: EUci, config_names: list[str]) -> str: + """ + Expand application groups into the parenthesised OR block of an action criteria. + + Members of several groups are merged and deduplicated; categories are never expanded into their + applications, so a category keeps following the signature updates. Groups that do not exist are + skipped. + + Args: + - e_uci: euci instance + - config_names: config names of the groups to expand + + Returns: + the expression, e.g. `(app == 'netify.amazon' || proto == 'http/connect')`, or an empty string + if the groups hold no member + """ + terms = [] + seen = set[str]() + for uci_list in APPGROUP_MEMBERS: + for config_name in config_names: + if e_uci.get('dpi', config_name, default=None) != 'appgroup': + continue + for value in e_uci.get('dpi', config_name, uci_list, list=True, default=[]): + # protocol and category names are matched lowercased, application tags already are + value = value if uci_list == 'app' else value.lower() + term = f"{uci_list} == '{value}'" + if term not in seen: + seen.add(term) + terms.append(term) + + if not terms: + return '' + return f"({' || '.join(terms)})" + + +# Rules: ordered by priority, first match wins. A rule matches application groups only, optionally +# narrowed to a source; what it does is decided by its action. + +# 'allow' attaches a label no nft rule matches: it exists so a rule can win the first-match race and +# shield its traffic from the block rules below it +DPI_RULE_ACTIONS = ('block', 'allow') + +DPI_RULE_NAME_MAX_LENGTH = 64 + +DPI_RULE_POSITIONS = ('top', 'bottom') + + +def __sorted_rules(e_uci: EUci) -> list[tuple[str, dict]]: + """ + Rule sections in priority order. Rules with no priority, i.e. written by hand, go last keeping + their config order. + """ + def sort_key(item: tuple[str, dict]): + try: + return 0, int(item[1].get('priority')) + except (TypeError, ValueError): + return 1, 0 + + return sorted((utils.get_all_by_type(e_uci, 'dpi', 'rule') or {}).items(), key=sort_key) + + +def __renumber_rules(e_uci: EUci, order: list[str] = None): + """ + Give every rule a dense priority, starting at 1. The generator keeps 0 for its own action, so no + user rule can be evaluated before it. + """ + if order is None: + order = [section for section, _ in __sorted_rules(e_uci)] + for priority, section in enumerate(order, start=1): + e_uci.set('dpi', section, 'priority', priority) + + +def __is_rule(e_uci: EUci, config_name: str) -> bool: + return e_uci.get('dpi', config_name, default=None) == 'rule' + + +def __validate_rule_name(name: str) -> str: + if not isinstance(name, str) or not name.strip(): + raise ValidationError('name', 'name_required', name) + name = name.strip() + if len(name) > DPI_RULE_NAME_MAX_LENGTH: + raise ValidationError('name', 'name_too_long', name) + return name + + +def __validate_source(values: list[str]) -> list[str]: + """ + Validate the source of a rule: addresses, networks and ranges, IPv4 and IPv6 alike. An empty + source means the rule matches every host. + """ + sources = [] + for value in values or []: + if not isinstance(value, str) or not value.strip(): + raise ValidationError('source', 'invalid_source', value) + source = value.strip() + try: + if '-' in source: + first, last = (ipaddress.ip_address(part.strip()) for part in source.split('-', 1)) + if first.version != last.version or first > last: + raise ValueError(source) + else: + ipaddress.ip_network(source, strict=False) + except ValueError: + raise ValidationError('source', 'invalid_source', value) + if source not in sources: + sources.append(source) + return sources + + +def __validate_appgroups(e_uci: EUci, config_names: list[str], require_appgroup: bool = True) -> list[str]: + """ + Validate the groups a rule matches. At least one is required by default: a rule with a source and + no group would be an IP-level rule, which the firewall does better, and a rule with neither would + expand to an empty criteria. + + `require_appgroup=False` lifts that requirement for a rule migrated from a global exemption: it is + a source-only Allow rule by design, a shape the drawer cannot produce and never will. + """ + appgroups = [] + for config_name in config_names or []: + if not isinstance(config_name, str) or e_uci.get('dpi', config_name, default=None) != 'appgroup': + raise ValidationError('appgroups', 'appgroup_does_not_exists', config_name) + if config_name not in appgroups: + appgroups.append(config_name) + if require_appgroup and not appgroups: + raise ValidationError('appgroups', 'appgroups_required', appgroups) + return appgroups + + +def expand_source(values: list[str]) -> list[str]: + """ + Expand the source of a rule into values a criteria can match: ranges become CIDR blocks, addresses + and networks are left as they are. + + Args: + - values: source entries of the rule + + Returns: + list of addresses and CIDR blocks + """ + expanded = [] + for value in values: + if '-' in value: + first, last = (ipaddress.ip_address(part.strip()) for part in value.split('-', 1)) + expanded.extend(str(network) for network in ipaddress.summarize_address_range(first, last)) + else: + expanded.append(value) + return expanded + + +def build_rule_criteria(e_uci: EUci, rule: dict, require_appgroup: bool = True) -> str: + """ + Build the criteria of a managed rule: the source, when set, ANDed with the union of its groups. + + Args: + - e_uci: euci instance + - rule: the rule section, as returned by `uci.get_all` + - require_appgroup: when False, a rule with a source and no group emits a source-only criteria + instead of being treated as matching nothing. Used for a rule migrated from a global exemption, + the one shape of source-only Allow rule the drawer cannot produce and never will. + + Returns: + the semicolon-terminated expression, or an empty string when the rule matches nothing and must + not be emitted + """ + match = expand_appgroups(e_uci, list(rule.get('appgroup', []))) + sources = [f'local_ip == {source}' for source in expand_source(list(rule.get('source', [])))] + + if not match: + if require_appgroup or not sources: + return '' + return f"({' || '.join(sources)});" + + if sources: + return f"({' || '.join(sources)}) && {match};" + return f'{match};' + + +def list_rules(e_uci: EUci) -> list[dict]: + """ + List the rules in priority order, the order they are evaluated in. + + Rules hidden with `ns_visible '0'` are left out: they are system entries the user must not touch. + + Args: + - e_uci: euci instance + + Returns: + list of dicts, each dict contains the property "id", "name", "enabled", "action", "source", + "appgroups", "managed" and "index", plus "criteria" for the rules the API did not create + """ + groups = {section: group.get('ns_name', '') + for section, group in (utils.get_all_by_type(e_uci, 'dpi', 'appgroup') or {}).items()} + rules = [] + for section, rule in __sorted_rules(e_uci): + if rule.get('ns_visible', '1') == '0': + continue + data_rule = { + 'id': section, + 'name': rule.get('ns_name', ''), + 'enabled': rule.get('enabled', '1') == '1', + 'action': rule.get('action', ''), + 'source': list(rule.get('source', [])), + 'appgroups': [{'id': group, 'name': groups.get(group, group)} + for group in rule.get('appgroup', [])], + # a rule the UI did not create can be renamed, toggled, reordered and deleted, but not edited + 'managed': rule.get('ns_managed', '0') == '1', + 'index': len(rules) + } + if 'criteria' in rule: + data_rule['criteria'] = rule.get('criteria') + rules.append(data_rule) return rules -def __save_rule_data(e_uci: EUci, config_name: str, enabled: bool, device: str, action: str, applications: list[str], - protocols: list[str]): +def __save_rule_data(e_uci: EUci, config_name: str, name: str, enabled: bool, action: str, + source: list[str], appgroups: list[str]): + e_uci.set('dpi', config_name, 'ns_name', name) + e_uci.set('dpi', config_name, 'ns_managed', '1') e_uci.set('dpi', config_name, 'enabled', enabled) - e_uci.set('dpi', config_name, 'device', device) e_uci.set('dpi', config_name, 'action', action) - e_uci.set('dpi', config_name, 'application', applications) - e_uci.set('dpi', config_name, 'protocol', protocols) + if appgroups: + e_uci.set('dpi', config_name, 'appgroup', appgroups) + else: + # a rule migrated from a global exemption has no group: the option must go, not stay empty + e_uci.delete('dpi', config_name, 'appgroup') + if source: + e_uci.set('dpi', config_name, 'source', source) + else: + # no source means every host: the option must go, not stay empty + e_uci.delete('dpi', config_name, 'source') -def __save_exemption_data(e_uci: EUci, config_name: str, criteria: str, description: str, enabled: bool): - e_uci.set('dpi', config_name, 'enabled', enabled) - e_uci.set('dpi', config_name, 'criteria', criteria) - e_uci.set('dpi', config_name, 'description', description) -def __toggle_engine(e_uci: EUci): - count_enabled = 0 - for section in e_uci.get_all('dpi'): - if e_uci.get('dpi', section, default="") == "rule" and e_uci.get('dpi', section, 'enabled', default="0") == "1": - count_enabled = count_enabled + 1 +def add_rule(e_uci: EUci, name: str, enabled: bool, action: str, source: list[str], + appgroups: list[str], position: str = 'bottom', require_appgroup: bool = True) -> str: + """ + Store a new rule. - if count_enabled > 0: - e_uci.set('dpi', 'config', 'enabled', '1') - else: - e_uci.set('dpi', 'config', 'enabled', '0') + Args: + - e_uci: euci instance + - name: name of the rule + - enabled: enable the rule + - action: 'block' or 'allow' + - source: list of addresses, networks or ranges, empty to match every host + - appgroups: config names of the application groups the rule matches, at least one unless + `require_appgroup` is False + - position: 'top' to evaluate the rule before every other one, 'bottom' after them + - require_appgroup: set to False only when migrating a global exemption into a source-only Allow + rule, a shape the drawer cannot produce and never will -def add_rule(e_uci: EUci, enabled: bool, device: str, action: str, applications: list[str], - protocols: list[str]) -> str: + Returns: + config name of the rule created + + Raises: + - ValidationError: if any argument is invalid """ - Store a new rule + name = __validate_rule_name(name) + if action not in DPI_RULE_ACTIONS: + raise ValidationError('action', 'invalid_action', action) + if position not in DPI_RULE_POSITIONS: + raise ValidationError('position', 'invalid_position', position) + source = __validate_source(source) + appgroups = __validate_appgroups(e_uci, appgroups, require_appgroup) + + config_name = utils.get_random_id() + e_uci.set('dpi', config_name, 'rule') + __save_rule_data(e_uci, config_name, name, enabled, action, source, appgroups) + + order = [section for section, _ in __sorted_rules(e_uci) if section != config_name] + order.insert(0, config_name) if position == 'top' else order.append(config_name) + __renumber_rules(e_uci, order) + + __toggle_engine(e_uci) + e_uci.save('dpi') + return config_name + + +def edit_rule(e_uci: EUci, config_name: str, name: str, enabled: bool, action: str, source: list[str], + appgroups: list[str]) -> str: + """ + Edit a rule. Only rules created through the API can be edited: a rule carrying a hand-written + criteria has no source and no group to fill the form with, and rewriting it would change what it + matches. Args: - e_uci: euci instance - - description: description of the rule + - config_name: config name of the rule to edit + - name: name of the rule - enabled: enable the rule - - action: apply specific action to rule, can be 'block', 'bulk', 'best_effort', 'video' or 'voice' - - device: device to listen and apply the rule on - - applications: list of applications to block - - protocols: list of protocols to block + - action: 'block' or 'allow' + - source: list of addresses, networks or ranges, empty to match every host + - appgroups: config names of the application groups the rule matches, at least one Returns: - config name of the rule created + config name of the rule edited + + Raises: + - ValidationError: if the rule does not exist, is not managed, or any argument is invalid """ - rule_name = utils.get_random_id() - e_uci.set('dpi', rule_name, 'rule') - __save_rule_data(e_uci, rule_name, enabled, device, action, applications, protocols) + if not __is_rule(e_uci, config_name): + raise ValidationError('id', 'rule_not_found', config_name) + if e_uci.get('dpi', config_name, 'ns_managed', default='0') != '1': + raise ValidationError('id', 'rule_not_managed', config_name) + + name = __validate_rule_name(name) + if action not in DPI_RULE_ACTIONS: + raise ValidationError('action', 'invalid_action', action) + source = __validate_source(source) + appgroups = __validate_appgroups(e_uci, appgroups) + + __save_rule_data(e_uci, config_name, name, enabled, action, source, appgroups) __toggle_engine(e_uci) e_uci.save('dpi') - return rule_name + return config_name -def delete_rule(e_uci: EUci, config_name: str): +def delete_rule(e_uci: EUci, config_name: str) -> str: """ - Delete a rule + Delete a rule and close the gap its priority leaves behind. Args: - e_uci: euci instance - config_name: config name of the rule to delete + + Returns: + config name of the rule deleted + + Raises: + - ValidationError: if the rule does not exist """ + if not __is_rule(e_uci, config_name): + raise ValidationError('id', 'rule_not_found', config_name) + e_uci.delete('dpi', config_name) + __renumber_rules(e_uci) __toggle_engine(e_uci) e_uci.save('dpi') + return config_name -def edit_rule(e_uci: EUci, config_name: str, enabled: bool, device: str, action: str, applications: list[str], - protocols: list[str]): +def rename_rule(e_uci: EUci, config_name: str, name: str) -> str: """ - Edit a rule + Rename a rule, managed or not. Args: - e_uci: euci instance - - config_name: rule to change - - enabled: enable the rule - - device: device to listen and apply the rule on - - action: apply specific action to rule, can be 'block', 'bulk', 'best_effort', 'video' or 'voice' - - applications: array of applications to block - - protocols: array of protocols to block + - config_name: config name of the rule to rename + - name: new name of the rule - Raises - - ValidationError: if the config name is invalid - """ - if e_uci.get('dpi', config_name, default=None) is None: - raise ValidationError('config-name', 'invalid', config_name) + Returns: + config name of the rule renamed - __save_rule_data(e_uci, config_name, enabled, device, action, applications, protocols) - __toggle_engine(e_uci) + Raises: + - ValidationError: if the rule does not exist or the name is invalid + """ + if not __is_rule(e_uci, config_name): + raise ValidationError('id', 'rule_not_found', config_name) + e_uci.set('dpi', config_name, 'ns_name', __validate_rule_name(name)) e_uci.save('dpi') + return config_name -def list_exemptions(e_uci: EUci) -> list[dict[str]]: + +def enable_rule(e_uci: EUci, config_name: str) -> str: """ - Index all global exemptions + Enable a rule, managed or not. Args: - e_uci: euci instance + - config_name: config name of the rule to enable Returns: - list of dicts, each dict contains the property "config-name", "description", "enabled", "criteria" - """ - exemptions = list[dict[str]]() - fetch_ex = utils.get_all_by_type(e_uci, 'dpi', 'exemption') + config name of the rule enabled - if not fetch_ex: - return exemptions - for ex_name in fetch_ex.keys(): - # get content of exemption - ex = fetch_ex[ex_name] - # prepare the data to append to rules - data_ex = dict[str]() - data_ex['config-name'] = ex_name - data_ex['enabled'] = ex.get('enabled', '1') == '1' - data_ex['criteria'] = ex.get('criteria', '') - data_ex['description'] = ex.get('description', '') - # append exemption - exemptions.append(data_ex) + Raises: + - ValidationError: if the rule does not exist + """ + if not __is_rule(e_uci, config_name): + raise ValidationError('id', 'rule_not_found', config_name) - return exemptions + e_uci.set('dpi', config_name, 'enabled', '1') + __toggle_engine(e_uci) + e_uci.save('dpi') + return config_name -def add_exemption(e_uci: EUci, criteria: str, description: str, enabled: bool): +def disable_rule(e_uci: EUci, config_name: str) -> str: """ - Store a new global exemption + Disable a rule, managed or not. Args: - e_uci: euci instance - - criteria: exemption criteria, usually it's an IP address - - description: description of the rule - - enabled: enable the exemption + - config_name: config name of the rule to disable Returns: - config name of the exemption created + config name of the rule disabled + + Raises: + - ValidationError: if the rule does not exist """ - ex_list = utils.get_all_by_type(e_uci, 'dpi', 'exemption') - for ex_name in ex_list: - ex = ex_list[ex_name] - if ex.get('criteria', '') == criteria: - raise ValidationError('criteria', 'criteria_already_exists', criteria) + if not __is_rule(e_uci, config_name): + raise ValidationError('id', 'rule_not_found', config_name) - ex_name = utils.get_random_id() - e_uci.set('dpi', ex_name, 'exemption') - __save_exemption_data(e_uci, ex_name, criteria, description, enabled) + e_uci.set('dpi', config_name, 'enabled', '0') + __toggle_engine(e_uci) e_uci.save('dpi') - return ex_name + return config_name -def delete_exemption(e_uci: EUci, config_name: str): +def order_rules(e_uci: EUci, order: list[str]) -> list[str]: """ - Delete a global exemption + Reorder every rule, renumbering the priorities densely from the given order. + + The whole set must be listed, hidden rules included: a partial order would silently move the rules + left out, which with a paginated list is a rule the caller never saw. Args: - e_uci: euci instance - - config_name: config name of the rule to delete + - order: config names of every rule, in the order they must be evaluated + + Returns: + the list of the ordered rules + + Raises: + - ValidationError: if the order does not name every rule exactly once """ - e_uci.delete('dpi', config_name) + rules = [section for section, _ in __sorted_rules(e_uci)] + for section in order: + if section not in rules: + raise ValidationError('order', 'rule_not_found', section) + if len(order) != len(set(order)) or len(order) != len(rules): + raise ValidationError('order', 'invalid_order', order) + + __renumber_rules(e_uci, order) e_uci.save('dpi') + return order + + +# Migration: one-time, in-place bump of /etc/config/dpi from the schema used before application groups +# existed. A rule from that schema carries `device`, `application`, `protocol` or `category` — fields +# only that API ever wrote, so their presence is a permanent, unambiguous marker regardless of whether +# `priority` was since assigned to it by an unrelated add/edit/delete. A global exemption is identified +# by its section type, which this migration removes once every exemption has become a rule. + +def freeze_legacy_criteria(e_uci: EUci, rule: dict) -> str: + """ + Build the criteria of a rule written before application groups existed, the way `dpi-config` has + always generated it: source, application, protocol and category turned into an expression narrowed + to the rule's device, with the VLAN rewrite applied when the device names a VLAN. Used by the + generator for a legacy rule the migration has not converted yet, and by the migration itself to + freeze a rule's behaviour verbatim before clearing those fields. + + Args: + - e_uci: euci instance + - rule: the rule section, as returned by `uci.get_all` + + Returns: + the semicolon-terminated expression + """ + device = rule.get('device', '*') + + if 'criteria' in rule: + # criteria has precedence over source, protocol, category and application + criteria = rule['criteria'].replace('"', "'") + else: + sources = [] + for source in rule.get('source', []): + if objects.is_object_id(source): + for ip in objects.get_object_ips(e_uci, source): + sources.append(f'local_ip == {ip}') + else: + sources.append(f'local_ip == {source}') + + applications = [] + for app in rule.get('application', []): + applications.append(f"app == '{app}'") + for proto in rule.get('protocol', []): + applications.append(f"proto == '{proto.lower()}'") + for cat in rule.get('category', []): + applications.append(f"category == '{cat.lower()}'") + + sources_s = ' or '.join(sources) + applications_s = ' or '.join(applications) + criteria = f"(iface_nfq_src == '{device}' or iface_nfq_dst == '{device}') && " + if len(sources) < 1: + criteria += f'({applications_s}) ;' + elif len(applications) < 1: + criteria += f'({sources_s}) ;' + else: + criteria += f'({sources_s}) && ({applications_s}) ;' + + vlan_id = None + base_if = None + for item in utils.get_all_by_type(e_uci, 'network', 'device').values(): + if item.get('vid', None) is not None and item.get('name', '') == device: + vlan_id = item.get('vid', None) + base_if = item.get('ifname', None) + break + if vlan_id is not None and base_if is not None: + criteria = f'vlan_id == {vlan_id} && {criteria}' -def edit_exemption(e_uci: EUci, config_name: str, criteria: str, description: str, enabled: bool): + return criteria + + +def __exemption_source(e_uci: EUci, criteria: str) -> list[str] | None: + """ + Turn a global exemption's criteria into a rule source. A firewall object is expanded into its + member addresses; a plain address, network or range is kept as is. + + Returns: + the source list, or None when the criteria is empty or does not resolve to at least one + address, telling the caller to fall back to an unmanaged rule carrying it verbatim """ - Edit a global exemption + if not criteria: + return None + if objects.is_object_id(criteria): + return objects.get_object_ips(e_uci, criteria) or None + try: + return __validate_source([criteria]) + except ValidationError: + return None + + +def migrate_schema(e_uci: EUci) -> bool: + """ + Migrate every rule and exemption from the schema used before application groups existed. + + Each legacy rule (`device`, `application`, `protocol` or `category` set) is turned into an unmanaged + rule with its behaviour frozen verbatim into `criteria` — see `freeze_legacy_criteria`. A rule whose + action is a retired QoS value, carries a per-rule exemption (a field no recent API ever wrote), or + ends up matching nothing is dropped instead, with a log line naming it. + + Each global exemption becomes an Allow rule at the top of the list: a managed, source-only rule + (`require_appgroup=False`) when its criteria is a plain address, CIDR or firewall object; an + unmanaged one carrying the criteria verbatim otherwise. Disabled exemptions become disabled rules. + The `exemption` section type, `firewall_exemption` and `popular_filters` are then removed. + + Safe to call unconditionally on every boot: a box with nothing left in the old schema returns False + without touching UCI. Args: - e_uci: euci instance - - config_name: rule to change - - criteria: exemption criteria, usually it's an IP address - - description: description of the rule - - enabled: enable the exemption - Raises - - ValidationError: if the config name is invalid + Returns: + True if the config was changed (and therefore committed and reloaded), False otherwise """ - if e_uci.get('dpi', config_name, default=None) is None: - raise ValidationError('config-name', 'invalid', config_name) + exemptions = utils.get_all_by_type(e_uci, 'dpi', 'exemption') or {} + rules = utils.get_all_by_type(e_uci, 'dpi', 'rule') or {} + legacy_fields = ('device', 'application', 'protocol', 'category') + legacy_rules = {section for section, rule in rules.items() + if any(field in rule for field in legacy_fields)} + + if not exemptions and not legacy_rules: + return False + + order = [] + + exemption_count = 0 + for section, exemption in exemptions.items(): + exemption_count += 1 + name = f'Migrated exception {exemption_count}' + enabled = exemption.get('enabled', '1') == '1' + criteria = (exemption.get('criteria') or '').strip() + source = __exemption_source(e_uci, criteria) + + e_uci.delete('dpi', section) + if source is not None: + rule_id = add_rule(e_uci, name, enabled, 'allow', source, [], require_appgroup=False) + else: + rule_id = utils.get_random_id() + e_uci.set('dpi', rule_id, 'rule') + e_uci.set('dpi', rule_id, 'ns_name', name) + e_uci.set('dpi', rule_id, 'enabled', '1' if enabled else '0') + e_uci.set('dpi', rule_id, 'action', 'allow') + e_uci.set('dpi', rule_id, 'criteria', criteria if criteria.endswith(';') else f'{criteria};') + order.append(rule_id) + + rule_count = 0 + for section, rule in rules.items(): + if section not in legacy_rules: + order.append(section) + continue - __save_exemption_data(e_uci, config_name, criteria, description, enabled) + action = rule.get('action') + if action != 'block': + syslog.syslog(syslog.LOG_WARNING, f"dpi migration: dropping {section}, unsupported action {action!r}") + e_uci.delete('dpi', section) + continue + if rule.get('exemption'): + syslog.syslog(syslog.LOG_WARNING, + f"dpi migration: dropping the per-rule exemption on {section}, no longer supported") + if not any(rule.get(field) for field in ('criteria', 'source', 'application', 'protocol', 'category')): + syslog.syslog(syslog.LOG_WARNING, f"dpi migration: dropping {section}, it matches nothing") + e_uci.delete('dpi', section) + continue + + rule_count += 1 + criteria = freeze_legacy_criteria(e_uci, rule) + for field in (*legacy_fields, 'source', 'exemption'): + e_uci.delete('dpi', section, field) + e_uci.set('dpi', section, 'criteria', criteria) + e_uci.set('dpi', section, 'ns_name', f'Migrated rule {rule_count}') + order.append(section) + + __renumber_rules(e_uci, order) + e_uci.delete('dpi', 'config', 'firewall_exemption') + e_uci.delete('dpi', 'config', 'popular_filters') + __toggle_engine(e_uci) e_uci.save('dpi') + __apply(e_uci) + return True diff --git a/packages/python3-nethsec/tests/test_dpi.py b/packages/python3-nethsec/tests/test_dpi.py index 2e049dba2..2bcdb8072 100644 --- a/packages/python3-nethsec/tests/test_dpi.py +++ b/packages/python3-nethsec/tests/test_dpi.py @@ -4,7 +4,7 @@ from euci import EUci from pytest_mock import MockFixture -from nethsec import dpi +from nethsec import dpi, utils from nethsec.utils import ValidationError applications_file = """ @@ -197,6 +197,18 @@ } } +application_output = """ + 133: netify.netflix + 10119: netify.linkedin + 10552: netify.tesla + 10195: netify.avira + 10194: netify.sophos + 10244: netify.bbc + 10362: netify.hulu + 10118: netify.lets-encrypt + 199: netify.snapchat +""" + protocol_output = """ 116: Warcraft3 117: LotusNotes @@ -257,11 +269,6 @@ list protocol 'HTTP/Connect' option device 'eth1' option enabled 1 - -config exemption exemp1 - option criteria '192.168.1.1' - option description 'my host' - option enabled 1 """ netifyd_config = """ @@ -349,6 +356,15 @@ """ +@pytest.fixture(autouse=True) +def mock_apply(mocker: MockFixture): + """ + add/delete of rules and appgroups commit the dpi config and reload the dpi service right away; + tests only care about the resulting uci state, not about actually reloading a system service. + """ + return mocker.patch('nethsec.dpi.__apply') + + @pytest.fixture def e_uci(tmp_path: pathlib.Path) -> EUci: conf_dir = tmp_path.joinpath('conf') @@ -376,6 +392,7 @@ def e_uci_with_data(e_uci: EUci): @pytest.fixture def mock_load(mocker): mocker.patch('nethsec.dpi.__load_applications', return_value=applications) + mocker.patch('nethsec.dpi.load_applications', return_value=applications) mocker.patch('nethsec.dpi.__load_application_categories', return_value=application_categories) mocker.patch('nethsec.dpi.load_protocols', return_value=protocols) mocker.patch('nethsec.dpi.__load_protocol_categories', return_value=protocol_categories) @@ -398,6 +415,13 @@ def test_load_protocol_categories(mocker: MockFixture, data: str): assert dpi.__load_protocol_categories() == protocol_categories +def test_load_applications_from_engine(mocker: MockFixture): + process_result = mocker.stub('subprocess_return') + process_result.stdout = bytes(application_output, 'utf-8') + mocker.patch('subprocess.run', return_value=process_result) + assert dpi.load_applications() == applications + + def test_load_protocols(mocker: MockFixture): process_result = mocker.stub('subprocess_return') process_result.stdout = bytes(protocol_output, 'utf-8') @@ -675,393 +699,805 @@ def test_index_applications_paginate(mock_load): } -def test_list_empty_rules(e_uci, mock_load): - assert dpi.list_rules(e_uci) == [] +# application groups +category_output = """ + 1: application: adult + 5: application: cybersecurity + 12: application: games + 5: protocol: games + 18: protocol: web +""" -def test_list_rules(e_uci_with_data, mock_load): - assert dpi.list_rules(e_uci_with_data) == [ - { - 'config-name': 'rule0', - 'enabled': True, - 'interface': 'GREEN_1', - 'device': 'eth0', - 'action': 'block', - 'criteria': [ - { - 'id': 10119, - 'name': 'netify.linkedin', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - }, - { - 'id': 199, - 'name': 'netify.snapchat', - 'type': 'application', - 'category': { - 'name': 'last' - } - }, - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - } - ] - }, - { - 'config-name': 'rule1', - 'enabled': False, - 'interface': 'GREEN_2', - 'device': 'eth4', - 'action': 'block', - 'criteria': [ - { - 'id': 10552, - 'name': 'netify.tesla', - 'type': 'application', - 'category': { - 'name': 'first-category' - } - } - ] - }, - { - 'config-name': 'rule3', - 'enabled': True, - 'interface': 'RED_1', - 'device': 'eth1', - 'action': 'video', - 'criteria': [ - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - } - ] - } - ] +categories = { + 'application': {1: 'adult', 5: 'cybersecurity', 12: 'games'}, + 'protocol': {5: 'games', 18: 'web'} +} -def test_store_rule(e_uci, mock_load): - rule_created = dpi.add_rule(e_uci, True, 'eth0', 'best_effort', - ['netify.linkedin', 'netify.avira', 'netify.netflix'], ['LotusNotes', 'SFlow']) - assert dpi.list_rules(e_uci) == [ - { - 'config-name': rule_created, - 'enabled': True, - 'interface': 'GREEN_1', - 'device': 'eth0', - 'action': 'best_effort', - 'criteria': [ - { - 'id': 10119, - 'name': 'netify.linkedin', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - }, - { - 'id': 10195, - 'name': 'netify.avira', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - }, - { - 'id': 133, - 'name': 'netify.netflix', - 'type': 'application', - 'category': { - 'name': 'unknown' - } - }, - { - 'id': 117, - 'name': 'LotusNotes', - 'type': 'protocol', - }, - { - 'id': 129, - 'name': 'SFlow', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - } - ] - } - ] - assert(e_uci.get("dpi", "config", "enabled") == "1") +@pytest.fixture +def mock_vocabularies(mocker): + """Every member kind resolves against the engine, the catalogs are never reached.""" + mocker.patch('nethsec.dpi.load_applications', return_value=applications) + mocker.patch('nethsec.dpi.load_protocols', return_value=protocols) + mocker.patch('nethsec.dpi.load_categories', return_value=categories) -def test_delete_rule(e_uci_with_data, mock_load): - dpi.delete_rule(e_uci_with_data, 'rule1') - dpi.delete_rule(e_uci_with_data, 'rule0') - assert dpi.list_rules(e_uci_with_data) == [ - { - 'config-name': 'rule3', - 'enabled': True, - 'interface': 'RED_1', - 'device': 'eth1', - 'action': 'video', - 'criteria': [ - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - } - ] - } - ] - dpi.delete_rule(e_uci_with_data, 'rule2') - dpi.delete_rule(e_uci_with_data, 'rule3') - assert(e_uci_with_data.get("dpi", "config", "enabled") == "0") +@pytest.fixture +def e_uci_appgroups(e_uci_with_data, mock_vocabularies): + return e_uci_with_data -def test_edit_rule(e_uci_with_data, mock_load): - dpi.edit_rule(e_uci_with_data, 'rule0', False, 'eth0', 'voice', [], - ['HTTP/Connect', 'LotusNotes']) - assert dpi.list_rules(e_uci_with_data) == [ - { - 'config-name': 'rule0', - 'enabled': False, - 'interface': 'GREEN_1', - 'device': 'eth0', - 'action': 'voice', - 'criteria': [ - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - }, - { - 'id': 117, - 'name': 'LotusNotes', - 'type': 'protocol', - } - ] - }, - { - 'config-name': 'rule1', - 'enabled': False, - 'interface': 'GREEN_2', - 'device': 'eth4', - 'action': 'block', - 'criteria': [ - { - 'id': 10552, - 'name': 'netify.tesla', - 'type': 'application', - 'category': { - 'name': 'first-category' - } - } - ] - }, - { - 'config-name': 'rule3', - 'enabled': True, - 'interface': 'RED_1', - 'device': 'eth1', - 'action': 'video', - 'criteria': [ - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - } - } - ] - } - ] +def test_load_categories(mocker: MockFixture): + process_result = mocker.stub('subprocess_return') + process_result.stdout = bytes(category_output, 'utf-8') + mocker.patch('subprocess.run', return_value=process_result) + assert dpi.load_categories() == categories + + +def test_add_appgroup(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix'], ['cybersecurity'], + ['Dropbox'], ['web']) + assert e_uci_appgroups.get('dpi', group_id) == 'appgroup' + assert e_uci_appgroups.get('dpi', group_id, 'ns_name') == 'Group one' + assert e_uci_appgroups.get('dpi', group_id, 'app', list=True) == ('netify.netflix',) + assert e_uci_appgroups.get('dpi', group_id, 'app_category', list=True) == ('cybersecurity',) + assert e_uci_appgroups.get('dpi', group_id, 'proto', list=True) == ('Dropbox',) + assert e_uci_appgroups.get('dpi', group_id, 'proto_category', list=True) == ('web',) + + +def test_add_appgroup_strips_and_deduplicates(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, ' Group one ', + [' netify.netflix ', 'netify.netflix', 'NETIFY.NETFLIX'], [], [], []) + assert e_uci_appgroups.get('dpi', group_id, 'ns_name') == 'Group one' + assert e_uci_appgroups.get('dpi', group_id, 'app', list=True) == ('netify.netflix',) + + +def test_add_appgroup_with_one_kind_only(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Only protocols', protocols=['HTTP/Connect']) + assert e_uci_appgroups.get('dpi', group_id, 'proto', list=True) == ('HTTP/Connect',) + assert e_uci_appgroups.get('dpi', group_id, 'app', default=None) is None + + +def test_add_appgroup_requires_a_name(e_uci_appgroups): + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, ' ', ['netify.netflix']) + assert err.value.args[0] == 'name' + assert err.value.args[1] == 'name_required' + + +def test_add_appgroup_refuses_a_long_name(e_uci_appgroups): + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'x' * 65, ['netify.netflix']) + assert err.value.args[1] == 'name_too_long' + + +def test_add_appgroup_refuses_a_duplicated_name(e_uci_appgroups): + dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'group ONE', ['netify.tesla']) + assert err.value.args[1] == 'name_already_exists' + + +def test_add_appgroup_refuses_an_empty_group(e_uci_appgroups): + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'Empty', [], [], [], []) + assert err.value.args[0] == 'members' + assert err.value.args[1] == 'appgroup_is_empty' + + +@pytest.mark.parametrize('value', ["netify.netflix'", 'netify.netflix;', 'netify.(netflix)', + 'netify.netflix\\', 'netify\nnetify.netflix', ' ']) +def test_add_appgroup_refuses_expression_breaking_members(e_uci_appgroups, value): + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'Group one', [value]) + assert err.value.args[0] == 'applications' + assert err.value.args[1] == 'invalid_application' + + +def test_add_appgroup_refuses_unknown_members(e_uci_appgroups): + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.nonexistent']) + assert err.value.args[1] == 'invalid_application' + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'Group one', protocols=['nonexistent']) + assert err.value.args[1] == 'invalid_protocol' + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'Group one', application_categories=['web']) + assert err.value.args[1] == 'invalid_application_category' + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'Group one', protocol_categories=['cybersecurity']) + assert err.value.args[1] == 'invalid_protocol_category' + + +def test_add_appgroup_refuses_a_protocol_named_by_its_catalog_tag(e_uci_appgroups): + # the catalog calls it "http-connect", but only the name the engine reports can be matched + with pytest.raises(ValidationError) as err: + dpi.add_appgroup(e_uci_appgroups, 'Group one', protocols=['http-connect']) + assert err.value.args[1] == 'invalid_protocol' + + +def test_add_appgroup_matches_members_case_insensitively(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', protocols=['http/connect']) + assert e_uci_appgroups.get('dpi', group_id, 'proto', list=True) == ('http/connect',) + + +def test_add_appgroup_skips_validation_without_a_vocabulary(e_uci_with_data, mocker): + mocker.patch('nethsec.dpi.load_applications', side_effect=FileNotFoundError) + mocker.patch('nethsec.dpi.__load_catalog_tags', side_effect=FileNotFoundError) + group_id = dpi.add_appgroup(e_uci_with_data, 'Group one', ['netify.unknown-to-everything']) + assert e_uci_with_data.get('dpi', group_id, 'app', list=True) == ('netify.unknown-to-everything',) + + +def test_edit_appgroup(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix'], ['cybersecurity']) + dpi.edit_appgroup(e_uci_appgroups, group_id, 'Group two', ['netify.tesla']) + assert e_uci_appgroups.get('dpi', group_id, 'ns_name') == 'Group two' + assert e_uci_appgroups.get('dpi', group_id, 'app', list=True) == ('netify.tesla',) + # an emptied list is removed, not stored empty + assert e_uci_appgroups.get('dpi', group_id, 'app_category', default=None) is None + + +def test_edit_appgroup_keeps_its_own_name(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + dpi.edit_appgroup(e_uci_appgroups, group_id, 'Group one', ['netify.tesla']) + assert e_uci_appgroups.get('dpi', group_id, 'app', list=True) == ('netify.tesla',) + + +def test_edit_appgroup_of_unknown_id(e_uci_appgroups): + with pytest.raises(ValidationError) as err: + dpi.edit_appgroup(e_uci_appgroups, 'ns_nonexistent', 'Group one', ['netify.netflix']) + assert err.value.args[1] == 'appgroup_does_not_exists' + + +def test_edit_appgroup_refuses_a_rule_id(e_uci_appgroups): + with pytest.raises(ValidationError) as err: + dpi.edit_appgroup(e_uci_appgroups, 'rule0', 'Group one', ['netify.netflix']) + assert err.value.args[1] == 'appgroup_does_not_exists' + + +def test_delete_appgroup(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + dpi.delete_appgroup(e_uci_appgroups, group_id) + assert e_uci_appgroups.get('dpi', group_id, default=None) is None + +def test_add_appgroup_leaves_the_change_pending(e_uci_appgroups, mock_apply): + dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + mock_apply.assert_not_called() -def test_edit_rule_with_missing_rule(e_uci): + +def test_delete_appgroup_leaves_the_change_pending(e_uci_appgroups, mock_apply): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + mock_apply.reset_mock() + dpi.delete_appgroup(e_uci_appgroups, group_id) + mock_apply.assert_not_called() + + +def test_delete_appgroup_of_unknown_id(e_uci_appgroups): + with pytest.raises(ValidationError) as err: + dpi.delete_appgroup(e_uci_appgroups, 'ns_nonexistent') + assert err.value.args[1] == 'appgroup_does_not_exists' + + +def test_delete_appgroup_in_use(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + e_uci_appgroups.set('dpi', 'rule0', 'appgroup', [group_id]) with pytest.raises(ValidationError) as err: - dpi.edit_rule(e_uci, 'rule0', False, 'eth0', 'block', [], []) - - assert err.value.args[0] == 'config-name' - assert err.value.args[1] == 'invalid' - assert err.value.args[2] == 'rule0' - - -def test_list_interfaces(e_uci_with_data): - assert dpi.list_devices(e_uci_with_data).index({ - 'interface': 'GREEN_1', - 'device': 'eth0' - }) != -1 - assert dpi.list_devices(e_uci_with_data).index({ - 'interface': 'GREEN_2', - 'device': 'eth4' - }) != -1 - with pytest.raises(ValueError): - assert dpi.list_devices(e_uci_with_data).index({ - 'interface': 'RED_1', - 'device': 'eth1' - }) - - -def test_list_popular(e_uci_with_data, mock_load): - assert dpi.list_popular(e_uci_with_data) == { + dpi.delete_appgroup(e_uci_appgroups, group_id) + assert err.value.args[1] == 'appgroup_is_used' + assert err.value.args[2] == ['dpi/rule0'] + + +def test_is_used_appgroup(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + assert dpi.is_used_appgroup(e_uci_appgroups, group_id) == (False, []) + e_uci_appgroups.set('dpi', 'rule0', 'appgroup', [group_id]) + e_uci_appgroups.set('dpi', 'rule1', 'appgroup', [group_id]) + used, matches = dpi.is_used_appgroup(e_uci_appgroups, group_id) + assert used + assert sorted(matches) == ['dpi/rule0', 'dpi/rule1'] + + +def test_list_appgroups(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix'], ['cybersecurity'], + ['Dropbox'], ['web']) + assert dpi.list_appgroups(e_uci_appgroups) == { 'data': [ { - 'id': 133, - 'name': 'netify.netflix', - 'type': 'application', - 'category': { - 'name': 'unknown' - }, - 'missing': False - }, - { - 'id': 10362, - 'name': 'netify.hulu', - 'type': 'application', - 'category': { - 'name': 'first-category' - }, - 'missing': False - }, - { - 'name': 'netify.whatsapp', - 'missing': True - }, - { - 'name': 'netify.facebook', - 'missing': True - }, - { - 'id': 10194, - 'name': 'netify.sophos', - 'type': 'application', - 'category': { - 'name': 'first-category' - }, - 'missing': False - }, - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - }, - 'missing': False - }, - { - 'id': 121, - 'name': 'Dropbox', - 'type': 'protocol', - 'category': { - 'name': 'base' - }, - 'missing': False + 'id': group_id, + 'name': 'Group one', + 'applications': ['netify.netflix'], + 'application_categories': ['cybersecurity'], + 'protocols': ['Dropbox'], + 'protocol_categories': ['web'], + 'used': False, + 'matches': [] } ], 'meta': { 'last_page': 1, - 'total': 7 + 'total': 1 } } -def test_list_popular_with_limits(e_uci_with_data, mock_load): - assert dpi.list_popular(e_uci_with_data, limit=2, page=2) == { - 'data': [ - { - 'name': 'netify.whatsapp', - 'missing': True - }, - { - 'name': 'netify.facebook', - 'missing': True - } - ], - 'meta': { - 'last_page': 4, - 'total': 7 - } - } - assert dpi.list_popular(e_uci_with_data, limit=2, page=3) == { - 'data': [ - { - 'id': 10194, - 'name': 'netify.sophos', - 'type': 'application', - 'category': { - 'name': 'first-category' - }, - 'missing': False - }, - { - 'id': 130, - 'name': 'HTTP/Connect', - 'type': 'protocol', - 'category': { - 'name': 'low' - }, - 'missing': False - } - ], - 'meta': { - 'last_page': 4, - 'total': 7 - } - } +def test_list_appgroups_when_empty(e_uci_appgroups): + assert dpi.list_appgroups(e_uci_appgroups) == {'data': [], 'meta': {'last_page': 1, 'total': 0}} + + +def test_list_appgroups_is_ordered_by_name(e_uci_appgroups): + dpi.add_appgroup(e_uci_appgroups, 'zeta', ['netify.netflix']) + dpi.add_appgroup(e_uci_appgroups, 'Alpha', ['netify.tesla']) + assert [group['name'] for group in dpi.list_appgroups(e_uci_appgroups)['data']] == ['Alpha', 'zeta'] + + +def test_list_appgroups_pagination(e_uci_appgroups): + for index in range(5): + dpi.add_appgroup(e_uci_appgroups, f'Group {index}', ['netify.netflix']) + result = dpi.list_appgroups(e_uci_appgroups, limit=2, page=2) + assert [group['name'] for group in result['data']] == ['Group 2', 'Group 3'] + assert result['meta'] == {'last_page': 3, 'total': 5} + + +def test_list_appgroups_search(e_uci_appgroups): + dpi.add_appgroup(e_uci_appgroups, 'Streaming', ['netify.netflix']) + dpi.add_appgroup(e_uci_appgroups, 'Business', ['netify.tesla']) + result = dpi.list_appgroups(e_uci_appgroups, search='STREAM') + assert [group['name'] for group in result['data']] == ['Streaming'] + assert result['meta'] == {'last_page': 1, 'total': 1} + + +def test_list_appgroups_reports_usage(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + e_uci_appgroups.set('dpi', 'rule0', 'appgroup', [group_id]) + group = dpi.list_appgroups(e_uci_appgroups)['data'][0] + assert group['used'] + assert group['matches'] == ['dpi/rule0'] + + +def test_expand_appgroups(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix'], ['cybersecurity'], + ['HTTP/Connect'], ['web']) + assert dpi.expand_appgroups(e_uci_appgroups, [group_id]) == ( + "(app == 'netify.netflix' || app_category == 'cybersecurity'" + " || proto == 'http/connect' || proto_category == 'web')" + ) + + +def test_expand_appgroups_merges_and_deduplicates(e_uci_appgroups): + first = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix'], [], ['Dropbox']) + second = dpi.add_appgroup(e_uci_appgroups, 'Group two', ['netify.netflix', 'netify.tesla']) + assert dpi.expand_appgroups(e_uci_appgroups, [first, second]) == ( + "(app == 'netify.netflix' || app == 'netify.tesla' || proto == 'dropbox')" + ) + + +def test_expand_appgroups_skips_unknown_groups(e_uci_appgroups): + group_id = dpi.add_appgroup(e_uci_appgroups, 'Group one', ['netify.netflix']) + assert dpi.expand_appgroups(e_uci_appgroups, [group_id, 'ns_nonexistent', 'rule0']) == ( + "(app == 'netify.netflix')" + ) + + +def test_expand_appgroups_without_groups(e_uci_appgroups): + assert dpi.expand_appgroups(e_uci_appgroups, []) == '' + + +# rules + +@pytest.fixture +def e_uci_rules(e_uci, mock_vocabularies): + """A clean dpi config with one application group to build rules on.""" + with pathlib.Path(e_uci.confdir()).joinpath('dpi').open('w') as fp: + fp.write(dpi_minimal_db) + return e_uci + + +def group_of(e_uci, name='Group one'): + return dpi.add_appgroup(e_uci, name, ['netify.netflix']) + + +def test_list_rules_when_empty(e_uci_rules): + assert dpi.list_rules(e_uci_rules) == [] + + +def test_add_rule(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', ['192.168.1.0/24'], [group]) + assert e_uci_rules.get('dpi', rule_id) == 'rule' + assert e_uci_rules.get('dpi', rule_id, 'ns_name') == 'Block streaming' + assert e_uci_rules.get('dpi', rule_id, 'ns_managed') == '1' + assert e_uci_rules.get('dpi', rule_id, 'enabled') == '1' + assert e_uci_rules.get('dpi', rule_id, 'action') == 'block' + assert e_uci_rules.get('dpi', rule_id, 'priority') == '1' + assert e_uci_rules.get('dpi', rule_id, 'source', list=True) == ('192.168.1.0/24',) + assert e_uci_rules.get('dpi', rule_id, 'appgroup', list=True) == (group,) + + +def test_add_rule_without_source(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Block everywhere', True, 'block', [], [group]) + assert e_uci_rules.get('dpi', rule_id, 'source', default=None) is None + + +def test_add_rule_enables_the_engine(e_uci_rules): + group = group_of(e_uci_rules) + dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) + assert e_uci_rules.get('dpi', 'config', 'enabled') == '1' + + +def test_add_rule_at_the_top(e_uci_rules): + group = group_of(e_uci_rules) + first = dpi.add_rule(e_uci_rules, 'First', True, 'block', [], [group]) + second = dpi.add_rule(e_uci_rules, 'Second', True, 'allow', [], [group], position='top') + assert [rule['id'] for rule in dpi.list_rules(e_uci_rules)] == [second, first] + assert e_uci_rules.get('dpi', second, 'priority') == '1' + assert e_uci_rules.get('dpi', first, 'priority') == '2' + + +def test_add_rule_requires_a_group(e_uci_rules): + with pytest.raises(ValidationError) as err: + dpi.add_rule(e_uci_rules, 'No match', True, 'block', ['192.168.1.1'], []) + assert err.value.args[0] == 'appgroups' + assert err.value.args[1] == 'appgroups_required' + + +def test_add_rule_refuses_an_unknown_group(e_uci_rules): + with pytest.raises(ValidationError) as err: + dpi.add_rule(e_uci_rules, 'Bad group', True, 'block', [], ['ns_nonexistent']) + assert err.value.args[1] == 'appgroup_does_not_exists' + + +def test_add_rule_refuses_an_unknown_action(e_uci_rules): + group = group_of(e_uci_rules) + with pytest.raises(ValidationError) as err: + dpi.add_rule(e_uci_rules, 'Bad action', True, 'bulk', [], [group]) + assert err.value.args[0] == 'action' + assert err.value.args[1] == 'invalid_action' + + +def test_add_rule_requires_a_name(e_uci_rules): + group = group_of(e_uci_rules) + with pytest.raises(ValidationError) as err: + dpi.add_rule(e_uci_rules, ' ', True, 'block', [], [group]) + assert err.value.args[1] == 'name_required' + + +@pytest.mark.parametrize('source', ['192.168.1.1', '192.168.1.0/24', '192.168.1.10-192.168.1.20', + '2001:db8::1', '2001:db8::/64']) +def test_add_rule_accepts_every_source_shape(e_uci_rules, source): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, f'Rule {source}', True, 'block', [source], [group]) + assert e_uci_rules.get('dpi', rule_id, 'source', list=True) == (source,) + + +@pytest.mark.parametrize('source', ['not-an-ip', '192.168.1.300', '192.168.1.20-192.168.1.10', + '192.168.1.1-2001:db8::1', '']) +def test_add_rule_refuses_an_invalid_source(e_uci_rules, source): + group = group_of(e_uci_rules) + with pytest.raises(ValidationError) as err: + dpi.add_rule(e_uci_rules, 'Bad source', True, 'block', [source], [group]) + assert err.value.args[0] == 'source' + assert err.value.args[1] == 'invalid_source' + + +def test_edit_rule(e_uci_rules): + group = group_of(e_uci_rules) + other = group_of(e_uci_rules, 'Group two') + rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', ['192.168.1.1'], [group]) + dpi.edit_rule(e_uci_rules, rule_id, 'Allow streaming', False, 'allow', [], [other]) + assert e_uci_rules.get('dpi', rule_id, 'ns_name') == 'Allow streaming' + assert e_uci_rules.get('dpi', rule_id, 'enabled') == '0' + assert e_uci_rules.get('dpi', rule_id, 'action') == 'allow' + assert e_uci_rules.get('dpi', rule_id, 'appgroup', list=True) == (other,) + # an emptied source is removed, the rule matches every host again + assert e_uci_rules.get('dpi', rule_id, 'source', default=None) is None + + +def test_edit_rule_keeps_the_priority(e_uci_rules): + group = group_of(e_uci_rules) + first = dpi.add_rule(e_uci_rules, 'First', True, 'block', [], [group]) + second = dpi.add_rule(e_uci_rules, 'Second', True, 'block', [], [group]) + dpi.edit_rule(e_uci_rules, first, 'First edited', True, 'block', [], [group]) + assert [rule['id'] for rule in dpi.list_rules(e_uci_rules)] == [first, second] + + +def test_edit_rule_of_unknown_id(e_uci_rules): + group = group_of(e_uci_rules) + with pytest.raises(ValidationError) as err: + dpi.edit_rule(e_uci_rules, 'ns_nonexistent', 'Rule', True, 'block', [], [group]) + assert err.value.args[1] == 'rule_not_found' + + +def test_edit_rule_refuses_an_unmanaged_rule(e_uci_with_data, mock_vocabularies): + group = dpi.add_appgroup(e_uci_with_data, 'Group one', ['netify.netflix']) + with pytest.raises(ValidationError) as err: + dpi.edit_rule(e_uci_with_data, 'rule0', 'Rule', True, 'block', [], [group]) + assert err.value.args[1] == 'rule_not_managed' + + +def test_delete_rule_closes_the_gap(e_uci_rules): + group = group_of(e_uci_rules) + first = dpi.add_rule(e_uci_rules, 'First', True, 'block', [], [group]) + second = dpi.add_rule(e_uci_rules, 'Second', True, 'block', [], [group]) + third = dpi.add_rule(e_uci_rules, 'Third', True, 'block', [], [group]) + dpi.delete_rule(e_uci_rules, second) + assert e_uci_rules.get('dpi', first, 'priority') == '1' + assert e_uci_rules.get('dpi', third, 'priority') == '2' + + +def test_add_rule_leaves_the_change_pending(e_uci_rules, mock_apply): + group = group_of(e_uci_rules) + mock_apply.reset_mock() + dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) + mock_apply.assert_not_called() + + +def test_delete_rule_leaves_the_change_pending(e_uci_rules, mock_apply): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) + mock_apply.reset_mock() + dpi.delete_rule(e_uci_rules, rule_id) + mock_apply.assert_not_called() + + +def test_delete_rule_of_unknown_id(e_uci_rules): + with pytest.raises(ValidationError) as err: + dpi.delete_rule(e_uci_rules, 'ns_nonexistent') + assert err.value.args[1] == 'rule_not_found' + + +def test_delete_last_enabled_rule_disables_the_engine(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Only one', True, 'block', [], [group]) + dpi.delete_rule(e_uci_rules, rule_id) + assert e_uci_rules.get('dpi', 'config', 'enabled') == '0' + + +def test_rename_and_toggle_rule(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) + dpi.rename_rule(e_uci_rules, rule_id, 'Renamed') + assert e_uci_rules.get('dpi', rule_id, 'ns_name') == 'Renamed' + dpi.disable_rule(e_uci_rules, rule_id) + assert e_uci_rules.get('dpi', rule_id, 'enabled') == '0' + assert e_uci_rules.get('dpi', 'config', 'enabled') == '0' + dpi.enable_rule(e_uci_rules, rule_id) + assert e_uci_rules.get('dpi', rule_id, 'enabled') == '1' + assert e_uci_rules.get('dpi', 'config', 'enabled') == '1' + + +def test_rename_and_toggle_work_on_unmanaged_rules(e_uci_with_data): + dpi.rename_rule(e_uci_with_data, 'rule0', 'Migrated rule') + assert e_uci_with_data.get('dpi', 'rule0', 'ns_name') == 'Migrated rule' + dpi.disable_rule(e_uci_with_data, 'rule0') + assert e_uci_with_data.get('dpi', 'rule0', 'enabled') == '0' + + +def test_order_rules(e_uci_rules): + group = group_of(e_uci_rules) + first = dpi.add_rule(e_uci_rules, 'First', True, 'block', [], [group]) + second = dpi.add_rule(e_uci_rules, 'Second', True, 'block', [], [group]) + third = dpi.add_rule(e_uci_rules, 'Third', True, 'block', [], [group]) + dpi.order_rules(e_uci_rules, [third, first, second]) + assert [rule['id'] for rule in dpi.list_rules(e_uci_rules)] == [third, first, second] + assert e_uci_rules.get('dpi', third, 'priority') == '1' + + +def test_order_rules_refuses_a_partial_order(e_uci_rules): + group = group_of(e_uci_rules) + first = dpi.add_rule(e_uci_rules, 'First', True, 'block', [], [group]) + dpi.add_rule(e_uci_rules, 'Second', True, 'block', [], [group]) + with pytest.raises(ValidationError) as err: + dpi.order_rules(e_uci_rules, [first]) + assert err.value.args[1] == 'invalid_order' -def test_list_exemptions(e_uci_with_data): - assert dpi.list_exemptions(e_uci_with_data) == [ - { - 'config-name': 'exemp1', - 'enabled': True, - 'criteria': '192.168.1.1', - 'description': 'my host', - } - ] -def test_add_exemption(e_uci): - ex_created = dpi.add_exemption(e_uci, "192.168.2.2", 'my host2', True) - assert dpi.list_exemptions(e_uci) == [ +def test_order_rules_refuses_an_unknown_rule(e_uci_rules): + group = group_of(e_uci_rules) + first = dpi.add_rule(e_uci_rules, 'First', True, 'block', [], [group]) + with pytest.raises(ValidationError) as err: + dpi.order_rules(e_uci_rules, [first, 'ns_nonexistent']) + assert err.value.args[1] == 'rule_not_found' + + +def test_list_rules(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', ['192.168.1.0/24'], [group]) + assert dpi.list_rules(e_uci_rules) == [ { - 'config-name': ex_created, + 'id': rule_id, + 'name': 'Block streaming', 'enabled': True, - 'criteria': "192.168.2.2", - 'description': 'my host2', + 'action': 'block', + 'source': ['192.168.1.0/24'], + 'appgroups': [{'id': group, 'name': 'Group one'}], + 'managed': True, + 'index': 0 } ] + + +def test_list_rules_reports_unmanaged_rules(e_uci_with_data): + rules = {rule['id']: rule for rule in dpi.list_rules(e_uci_with_data)} + assert rules['rule0']['managed'] is False + assert rules['rule2']['criteria'] == 'local_ip == 192.168.100.22 && application == "netify.facebook";' + assert 'criteria' not in rules['rule3'] + + +def test_list_rules_hides_invisible_ones(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Hidden', True, 'block', [], [group]) + e_uci_rules.set('dpi', rule_id, 'ns_visible', '0') + assert dpi.list_rules(e_uci_rules) == [] + + +def test_list_rules_is_complete_and_indexed(e_uci_rules): + group = group_of(e_uci_rules) + for index in range(5): + dpi.add_rule(e_uci_rules, f'Rule {index}', True, 'block', [], [group]) + rules = dpi.list_rules(e_uci_rules) + assert [rule['name'] for rule in rules] == ['Rule 0', 'Rule 1', 'Rule 2', 'Rule 3', 'Rule 4'] + assert [rule['index'] for rule in rules] == [0, 1, 2, 3, 4] + + +def test_expand_source(): + assert dpi.expand_source(['192.168.1.1', '192.168.1.0/24']) == ['192.168.1.1', '192.168.1.0/24'] + assert dpi.expand_source(['192.168.1.10-192.168.1.11']) == ['192.168.1.10/31'] + assert dpi.expand_source(['192.168.1.1-192.168.1.4']) == ['192.168.1.1/32', '192.168.1.2/31', + '192.168.1.4/32'] + assert dpi.expand_source(['10.0.0.1-10.0.0.2']) == ['10.0.0.1/32', '10.0.0.2/32'] + + +def test_build_rule_criteria(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', + ['192.168.1.0/24', '10.0.0.1-10.0.0.2'], [group]) + rule = e_uci_rules.get_all('dpi', rule_id) + assert dpi.build_rule_criteria(e_uci_rules, rule) == ( + "(local_ip == 192.168.1.0/24 || local_ip == 10.0.0.1/32 || local_ip == 10.0.0.2/32)" + " && (app == 'netify.netflix');" + ) + + +def test_build_rule_criteria_without_source(e_uci_rules): + group = group_of(e_uci_rules) + rule_id = dpi.add_rule(e_uci_rules, 'Block streaming', True, 'block', [], [group]) + rule = e_uci_rules.get_all('dpi', rule_id) + assert dpi.build_rule_criteria(e_uci_rules, rule) == "(app == 'netify.netflix');" + + +def test_build_rule_criteria_of_a_rule_matching_nothing(e_uci_rules): + assert dpi.build_rule_criteria(e_uci_rules, {'source': ['192.168.1.1']}) == '' + + +def test_build_rule_criteria_source_only_when_appgroup_not_required(e_uci_rules): + rule = {'source': ['192.168.1.1', '192.168.1.0/24']} + assert dpi.build_rule_criteria(e_uci_rules, rule, require_appgroup=False) == ( + '(local_ip == 192.168.1.1 || local_ip == 192.168.1.0/24);' + ) + + +def test_build_rule_criteria_still_empty_without_source_or_appgroup(e_uci_rules): + assert dpi.build_rule_criteria(e_uci_rules, {}, require_appgroup=False) == '' + + +def test_add_rule_without_appgroup_requires_the_flag(e_uci_rules): with pytest.raises(ValidationError) as err: - dpi.add_exemption(e_uci, "192.168.2.2", 'duplicated', True) + dpi.add_rule(e_uci_rules, 'No group', True, 'allow', ['192.168.1.1'], []) + assert err.value.args[1] == 'appgroups_required' + + +def test_add_rule_source_only_via_migration_flag(e_uci_rules): + rule_id = dpi.add_rule(e_uci_rules, 'Migrated exception 1', True, 'allow', ['192.168.1.1'], [], + require_appgroup=False) + assert e_uci_rules.get('dpi', rule_id, 'ns_managed') == '1' + assert e_uci_rules.get('dpi', rule_id, 'source', list=True) == ('192.168.1.1',) + assert e_uci_rules.get('dpi', rule_id, 'appgroup', default=None) is None + + +# Migration from the schema used before application groups existed + +def test_freeze_legacy_criteria_from_application_and_device(e_uci: EUci): + rule = {'device': 'eth0', 'application': ['netify.amazon-prime', 'netify.netflix']} + assert dpi.freeze_legacy_criteria(e_uci, rule) == ( + "(iface_nfq_src == 'eth0' or iface_nfq_dst == 'eth0') && " + "(app == 'netify.amazon-prime' or app == 'netify.netflix') ;" + ) + + +def test_freeze_legacy_criteria_prefers_hand_written_criteria(e_uci: EUci): + rule = {'device': 'eth0', 'criteria': 'app == "netify.netflix";', 'application': ['netify.amazon-prime']} + assert dpi.freeze_legacy_criteria(e_uci, rule) == "app == 'netify.netflix';" + + +def test_freeze_legacy_criteria_applies_vlan_rewrite(e_uci: EUci): + with pathlib.Path(e_uci.confdir()).joinpath('network').open('a') as fp: + fp.write(""" +config device + option name 'eth0.10' + option ifname 'eth0' + option type '8021q' + option vid '10' +""") + rule = {'device': 'eth0.10', 'application': ['netify.netflix']} + assert dpi.freeze_legacy_criteria(e_uci, rule) == ( + "vlan_id == 10 && (iface_nfq_src == 'eth0.10' or iface_nfq_dst == 'eth0.10') && " + "(app == 'netify.netflix') ;" + ) + + +legacy_dpi_db = """ +config main 'config' + option log_blocked '0' + option firewall_exemption '1' + option enabled '0' + list popular_filters 'netify.facebook' + +config rule 'ns_b01a0e73' + option enabled '1' + option device 'eth0' + option action 'block' + list application 'netify.amazon-prime' + list application 'netify.netflix' + +config rule 'ns_qos1' + option enabled '1' + option device 'eth1' + option action 'bulk' + list application 'netify.dropbox' + +config rule 'ns_empty1' + option enabled '1' + option device 'eth2' + option action 'block' + +config rule 'ns_perrule_exempt' + option enabled '1' + option device 'eth3' + option action 'block' + list application 'netify.tesla' + list exemption '192.168.100.3' + +config exemption 'ns_2127b876' + option enabled '1' + option criteria '192.168.122.47' + option description 'my exception' + +config exemption 'ns_disabled_exempt' + option enabled '0' + option criteria '10.0.0.5' + option description 'disabled one' + +config exemption 'ns_bad_exempt' + option enabled '1' + option criteria 'not-an-address' + option description '' +""" + + +@pytest.fixture +def e_uci_legacy(e_uci: EUci): + with pathlib.Path(e_uci.confdir()).joinpath('dpi').open('w') as fp: + fp.write(legacy_dpi_db) + return e_uci + + +def test_migrate_schema_noop_when_nothing_to_migrate(e_uci, mock_apply): + assert dpi.migrate_schema(e_uci) is False + mock_apply.assert_not_called() + + +def test_migrate_schema_freezes_legacy_rule(e_uci_legacy): + assert dpi.migrate_schema(e_uci_legacy) is True + rule = e_uci_legacy.get_all('dpi', 'ns_b01a0e73') + assert rule['criteria'] == ( + "(iface_nfq_src == 'eth0' or iface_nfq_dst == 'eth0') && " + "(app == 'netify.amazon-prime' or app == 'netify.netflix') ;" + ) + assert 'device' not in rule + assert 'application' not in rule + assert 'ns_managed' not in rule + assert rule['action'] == 'block' + assert 'priority' in rule + + +def test_migrate_schema_names_migrated_rules_and_exemptions_in_order(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + assert e_uci_legacy.get('dpi', 'ns_b01a0e73', 'ns_name') == 'Migrated rule 1' + # ns_qos1 is dropped (QoS), ns_empty1 is dropped (matches nothing): ns_perrule_exempt is next + perrule_ids = [s for s, r in utils.get_all_by_type(e_uci_legacy, 'dpi', 'rule').items() + if r.get('ns_name') == 'Migrated rule 2'] + assert len(perrule_ids) == 1 + + +def test_migrate_schema_drops_qos_rule(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + assert e_uci_legacy.get('dpi', 'ns_qos1', default=None) is None + + +def test_migrate_schema_drops_rule_matching_nothing(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + assert e_uci_legacy.get('dpi', 'ns_empty1', default=None) is None + + +def test_migrate_schema_drops_per_rule_exemption_but_keeps_the_rule(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + rule = e_uci_legacy.get_all('dpi', 'ns_perrule_exempt') + assert 'exemption' not in rule + assert rule['action'] == 'block' + assert 'criteria' in rule + + +def test_migrate_schema_converts_address_exemption_to_managed_allow_rule(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + rules = dpi.list_rules(e_uci_legacy) + exception1 = next(r for r in rules if r['name'] == 'Migrated exception 1') + assert exception1['managed'] is True + assert exception1['action'] == 'allow' + assert exception1['source'] == ['192.168.122.47'] + assert exception1['appgroups'] == [] + assert exception1['enabled'] is True + + +def test_migrate_schema_keeps_a_disabled_exemption_disabled(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + rules = dpi.list_rules(e_uci_legacy) + exception2 = next(r for r in rules if r['name'] == 'Migrated exception 2') + assert exception2['enabled'] is False + + +def test_migrate_schema_converts_non_address_exemption_to_unmanaged_allow_rule(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + rules = dpi.list_rules(e_uci_legacy) + exception3 = next(r for r in rules if r['name'] == 'Migrated exception 3') + assert exception3['managed'] is False + assert exception3['action'] == 'allow' + assert exception3.get('criteria') == 'not-an-address;' + + +def test_migrate_schema_converts_object_exemption_via_expanded_addresses(e_uci, mocker): + with pathlib.Path(e_uci.confdir()).joinpath('dpi').open('w') as fp: + fp.write(""" +config main 'config' + option enabled '0' + +config exemption 'ns_objex' + option enabled '1' + option criteria 'objects/ns_obj1' + option description 'office subnet' +""") + mocker.patch('nethsec.dpi.objects.is_object_id', return_value=True) + mocker.patch('nethsec.dpi.objects.get_object_ips', return_value=['192.168.50.0/24']) + assert dpi.migrate_schema(e_uci) is True + rules = dpi.list_rules(e_uci) + assert len(rules) == 1 + assert rules[0]['managed'] is True + assert rules[0]['source'] == ['192.168.50.0/24'] + + +def test_migrate_schema_removes_exemption_sections_and_legacy_globals(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + assert utils.get_all_by_type(e_uci_legacy, 'dpi', 'exemption') == {} + assert e_uci_legacy.get('dpi', 'config', 'firewall_exemption', default=None) is None + assert e_uci_legacy.get('dpi', 'config', 'popular_filters', list=True, default=None) is None + + +def test_migrate_schema_puts_exemptions_before_migrated_rules(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + rules = dpi.list_rules(e_uci_legacy) + names_in_order = [r['name'] for r in rules] + assert names_in_order.index('Migrated exception 1') < names_in_order.index('Migrated rule 1') + + +def test_migrate_schema_applies_immediately(e_uci_legacy, mock_apply): + assert dpi.migrate_schema(e_uci_legacy) is True + mock_apply.assert_called_with(e_uci_legacy) -def test_edit_exemption(e_uci_with_data): - dpi.edit_exemption(e_uci_with_data, 'exemp1', '192.168.1.3', 'my host 3', False) - assert dpi.list_exemptions(e_uci_with_data) == [ - { - 'config-name': 'exemp1', - 'enabled': False, - 'criteria': "192.168.1.3", - 'description': 'my host 3', - } - ] -def test_delete_exemption(e_uci): - dpi.delete_exemption(e_uci, 'exemp1') - assert dpi.list_exemptions(e_uci) == [] +def test_migrate_schema_is_idempotent(e_uci_legacy): + dpi.migrate_schema(e_uci_legacy) + rules_after_first_run = dpi.list_rules(e_uci_legacy) + assert dpi.migrate_schema(e_uci_legacy) is False + assert dpi.list_rules(e_uci_legacy) == rules_after_first_run