From f2fa582f489ecb9d6371c582a51a1196085a5c18 Mon Sep 17 00:00:00 2001 From: Olivier Vernin Date: Tue, 4 Aug 2026 20:38:22 +0200 Subject: [PATCH 1/3] feat: expose endpoint with result history Signed-off-by: Olivier Vernin --- CONTRIBUTING.md | 1 + docs/docs.go | 572 +++++++++++++++++- docs/swagger.json | 572 +++++++++++++++++- docs/swagger.yaml | 430 ++++++++++++- pkg/database/database_test.go | 76 +++ ...elineReports_denormalized_columns.down.sql | 7 + ...ipelineReports_denormalized_columns.up.sql | 24 + pkg/database/report.go | 402 ++++++++++-- pkg/database/time_utils.go | 69 ++- pkg/server/endpoints.go | 2 + pkg/server/endpoints_test.go | 450 +++++++++++++- pkg/server/labeldb_handlers.go | 55 +- pkg/server/report_handlers.go | 178 ++++++ pkg/server/var.go | 33 +- 14 files changed, 2741 insertions(+), 130 deletions(-) create mode 100644 pkg/database/migrations/000010_fix_pipelineReports_denormalized_columns.down.sql create mode 100644 pkg/database/migrations/000010_fix_pipelineReports_denormalized_columns.up.sql diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1748bb4b..c809db4d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -51,6 +51,7 @@ The file `pkg/server/main.go` contains the following endpoint: * `/api/pipeline/scms`[GET] * `/api/pipeline/reports`[GET][POST] * `/api/pipeline/reports/:id`[GET][PUT][DELETE] +* `/api/pipeline/reports/summary`[POST] ## 3. Frontend diff --git a/docs/docs.go b/docs/docs.go index d5634be6..669b9c37 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -441,6 +441,131 @@ const docTemplate = `{ } } }, + "/api/pipeline/labels": { + "get": { + "description": "List labels data from the database with optional filtering", + "tags": [ + "Labels" + ], + "summary": "List labels", + "parameters": [ + { + "type": "string", + "description": "Filter by label ID", + "name": "id", + "in": "query" + }, + { + "type": "string", + "description": "Filter by label key", + "name": "key", + "in": "query" + }, + { + "type": "string", + "description": "Filter by label value", + "name": "value", + "in": "query" + }, + { + "type": "string", + "description": "Return only unique label keys (true/false)", + "name": "keyonly", + "in": "query" + }, + { + "type": "string", + "description": "Limit the number of labels returned, default is 100", + "name": "limit", + "in": "query" + }, + { + "type": "string", + "description": "Page number for pagination, default is 1", + "name": "page", + "in": "query" + }, + { + "type": "string", + "description": "Start time for filtering labels (RFC3339 format)", + "name": "start_time", + "in": "query" + }, + { + "type": "string", + "description": "End time for filtering labels (RFC3339 format)", + "name": "end_time", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ListLabelsResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + } + } + } + }, + "/api/pipeline/labels/search": { + "post": { + "description": "Search labels in the database using advanced filtering", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Labels" + ], + "summary": "Search labels", + "parameters": [ + { + "description": "Search parameters", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/server.SearchLabelsRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ListLabelsResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + } + } + } + }, "/api/pipeline/reports": { "get": { "description": "List all pipeline reports from the database", @@ -520,6 +645,12 @@ const docTemplate = `{ "$ref": "#/definitions/server.CreatePipelineReportResponse" } }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -578,6 +709,52 @@ const docTemplate = `{ } } }, + "/api/pipeline/reports/summary": { + "post": { + "description": "Return the number of pipeline reports per result for each time bucket of the requested time range.\nBuckets are UTC hours, UTC calendar days, ISO weeks or calendar months depending on the\ngranularity, and the date of an entry is the start of its bucket, formatted as RFC3339.\nEvery report is counted, including several reports of the same pipeline, and buckets without\nany report are returned with a zeroed entry.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Pipeline Reports" + ], + "summary": "Summarize pipeline reports", + "parameters": [ + { + "description": "Summary filters", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/server.SearchPipelineReportsSummaryRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.SearchPipelineReportsSummaryResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + } + } + } + }, "/api/pipeline/reports/{id}": { "get": { "description": "Get the latest pipeline report for a specific ID", @@ -595,8 +772,8 @@ const docTemplate = `{ } ], "responses": { - "201": { - "description": "Created", + "200": { + "description": "OK", "schema": { "$ref": "#/definitions/server.GetPipelineReportByIDResponse" } @@ -661,8 +838,8 @@ const docTemplate = `{ } ], "responses": { - "201": { - "description": "Created", + "200": { + "description": "OK", "schema": { "$ref": "#/definitions/server.DefaultResponseModel" } @@ -736,6 +913,58 @@ const docTemplate = `{ "responses": { "200": { "description": "OK", + "schema": { + "$ref": "#/definitions/server.ListSCMsResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + } + } + } + }, + "/api/pipeline/scms/search": { + "post": { + "description": "Search SCM data using JSON filters. When summary is true, the response contains SCM summary data for all matching SCMs.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "SCMs" + ], + "summary": "Search SCMs", + "parameters": [ + { + "description": "SCM search filters", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/server.SearchSCMsRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ListSCMsResponse" + } + }, + "400": { + "description": "Bad Request", "schema": { "$ref": "#/definitions/server.DefaultResponseModel" } @@ -812,9 +1041,37 @@ const docTemplate = `{ } } }, + "database.ReportResultSummaryEntry": { + "type": "object", + "properties": { + "date": { + "description": "Date is the start of the bucket, in UTC, formatted as RFC3339.", + "type": "string" + }, + "results": { + "description": "Results contains the number of reports per Updatecli result for that bucket.", + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "total": { + "description": "Total is the number of reports for that bucket, all results combined.", + "type": "integer" + } + } + }, "database.SearchLatestReportData": { "type": "object", "properties": { + "conditionConfigIDs": { + "description": "ConditionConfigIDs contains the config condition IDs associated with the report.", + "allOf": [ + { + "$ref": "#/definitions/pgtype.Hstore" + } + ] + }, "createdAt": { "description": "CreatedAt represents the creation date of the report.", "type": "string" @@ -843,6 +1100,22 @@ const docTemplate = `{ "description": "Result represents the result of the report.", "type": "string" }, + "sourceConfigIDs": { + "description": "SourceConfigIDs contains the config source IDs associated with the report.", + "allOf": [ + { + "$ref": "#/definitions/pgtype.Hstore" + } + ] + }, + "targetConfigIDs": { + "description": "TargetConfigIDs contains the config target IDs associated with the report.", + "allOf": [ + { + "$ref": "#/definitions/pgtype.Hstore" + } + ] + }, "updatedAt": { "description": "UpdatedAt represents the last update date of the report.", "type": "string" @@ -926,15 +1199,45 @@ const docTemplate = `{ } } }, + "model.Label": { + "type": "object", + "properties": { + "created_at": { + "description": "CreatedAt is the time the label was created", + "type": "string" + }, + "id": { + "description": "ID is a unique identifier for the label, generated as a UUID.", + "type": "string" + }, + "key": { + "description": "Key is the label name", + "type": "string" + }, + "last_pipeline_report_at": { + "description": "LastPipelineReportAt is the time the label was last used in a pipeline report", + "type": "string" + }, + "updated_at": { + "description": "UpdatedAt is the time the label was last updated", + "type": "string" + }, + "value": { + "description": "Value is the value associated with the label", + "type": "string" + } + } + }, "model.PipelineReport": { "type": "object", "properties": { "conditionConfigIDs": { "description": "ConditionConfigIDs is a list of unique identifiers of the condition configuration associated with the database.", - "type": "object", - "additionalProperties": { - "type": "string" - } + "allOf": [ + { + "$ref": "#/definitions/pgtype.Hstore" + } + ] }, "created_at": { "description": "Create_at represent the creation date of the record.", @@ -944,6 +1247,13 @@ const docTemplate = `{ "description": "ID is the unique identifier of the record in the database.", "type": "string" }, + "labelIDs": { + "description": "LabelIDs is a list of unique identifiers of the labels associated with the database.", + "type": "array", + "items": { + "type": "string" + } + }, "pipeline": { "description": "Pipeline represent the Updatecli pipeline report.", "allOf": [ @@ -966,17 +1276,19 @@ const docTemplate = `{ }, "sourceConfigIDs": { "description": "SourceConfigIDs is a list of unique identifiers of the source configuration associated with the database.", - "type": "object", - "additionalProperties": { - "type": "string" - } + "allOf": [ + { + "$ref": "#/definitions/pgtype.Hstore" + } + ] }, "targetConfigIDs": { "description": "TargetConfigIDs is a list of unique identifiers of the target configuration associated with the database.", - "type": "object", - "additionalProperties": { - "type": "string" - } + "allOf": [ + { + "$ref": "#/definitions/pgtype.Hstore" + } + ] }, "targetScmIDs": { "description": "TargetScmIDs is a list of unique identifiers of the scm configuration associated with the database.", @@ -991,6 +1303,37 @@ const docTemplate = `{ } } }, + "model.SCM": { + "type": "object", + "properties": { + "branch": { + "description": "Branch is the Git branch", + "type": "string" + }, + "created_at": { + "description": "Created_at is the time the SCM configuration was created", + "type": "string" + }, + "id": { + "description": "ID is a unique identifier for the SCM configuration", + "type": "string" + }, + "updated_at": { + "description": "Updated_at is the time the SCM configuration was last updated", + "type": "string" + }, + "url": { + "description": "URL is the Git repository URL", + "type": "string" + } + } + }, + "pgtype.Hstore": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, "reports.Action": { "type": "object", "properties": { @@ -1064,6 +1407,17 @@ const docTemplate = `{ } } }, + "reports.CIData": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, "reports.PipelineURL": { "type": "object", "properties": { @@ -1086,6 +1440,9 @@ const docTemplate = `{ "$ref": "#/definitions/reports.Action" } }, + "ci": { + "$ref": "#/definitions/reports.CIData" + }, "conditions": { "type": "object", "additionalProperties": { @@ -1102,6 +1459,12 @@ const docTemplate = `{ "description": "ID defines the report ID", "type": "string" }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, "name": { "type": "string" }, @@ -1445,6 +1808,183 @@ const docTemplate = `{ } } }, + "server.ListLabelsResponse": { + "type": "object", + "properties": { + "labels": { + "description": "Labels is a list of labels.", + "type": "array", + "items": { + "$ref": "#/definitions/model.Label" + } + }, + "total_count": { + "description": "TotalCount is the total number of labels matching the query.", + "type": "integer" + } + } + }, + "server.ListSCMsResponse": { + "type": "object", + "properties": { + "scms": { + "description": "SCMs is a list of SCMs.", + "type": "array", + "items": { + "$ref": "#/definitions/model.SCM" + } + }, + "total_count": { + "description": "TotalCount is the total number of SCMs matching the query.", + "type": "integer" + } + } + }, + "server.SearchLabelsRequest": { + "type": "object", + "properties": { + "end_time": { + "description": "EndTime is the end time for the time range filter\nThis is optional and can be used to filter labels by a specific end time\nTime format is RFC3339: 2006-01-02T15:04:05Z07:00", + "type": "string" + }, + "id": { + "description": "Id is the unique identifier of the label.", + "type": "string" + }, + "key": { + "description": "Key is the key of the label.", + "type": "string" + }, + "key_only": { + "description": "KeyOnly specifies if we only need to retrieve a list of uniq Label keys", + "type": "boolean" + }, + "limit": { + "description": "Limit is the maximum number of labels to return\nThis is optional and can be used to limit the number of labels returned", + "type": "integer" + }, + "page": { + "description": "Page is the page number for pagination\nThis is optional and can be used to paginate the results", + "type": "integer" + }, + "start_time": { + "description": "StartTime is the start time for the time range filter\nThis is optional and can be used to filter labels by a specific start time\nTime format is RFC3339: 2006-01-02T15:04:05Z07:00", + "type": "string" + }, + "value": { + "description": "Value is the value of the label.", + "type": "string" + } + } + }, + "server.SearchPipelineReportsSummaryRequest": { + "type": "object", + "properties": { + "days": { + "description": "Days is the number of days to summarize, today included.\nIt defaults to 7 and is ignored when hours, or start_time and end_time, are provided.", + "type": "integer" + }, + "end_time": { + "description": "EndTime is the end time for the time range filter.\nTime format is: 2006-01-02 15:04:05Z07:00", + "type": "string" + }, + "granularity": { + "description": "Granularity is the size of the time buckets, one of \"hour\", \"day\", \"week\" or\n\"month\". It defaults to \"day\".", + "type": "string" + }, + "hours": { + "description": "Hours is the number of hours to summarize, the current hour included.\nIt cannot be combined with days and is ignored when start_time and end_time are provided.", + "type": "integer" + }, + "labels": { + "description": "Labels is a map of labels to filter reports by.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "metric": { + "description": "Metric is what the reports are counted by. It defaults to \"result\", which is\nthe only value supported so far.", + "type": "string" + }, + "scmid": { + "description": "ScmID is the ID of the SCM to filter reports by.\nUse \"none\" to only count the reports which are not attached to any SCM.", + "type": "string" + }, + "start_time": { + "description": "StartTime is the start time for the time range filter.\nTime format is: 2006-01-02 15:04:05Z07:00", + "type": "string" + } + } + }, + "server.SearchPipelineReportsSummaryResponse": { + "type": "object", + "properties": { + "data": { + "description": "Data contains one entry per time bucket, ordered from the oldest to the most recent one.", + "type": "array", + "items": { + "$ref": "#/definitions/database.ReportResultSummaryEntry" + } + }, + "granularity": { + "description": "Granularity is the size of the time buckets of the entries.", + "type": "string" + }, + "metric": { + "description": "Metric is the metric the reports were counted by.", + "type": "string" + }, + "total_count": { + "description": "TotalCount is the total number of reports matching the query.", + "type": "integer" + } + } + }, + "server.SearchSCMsRequest": { + "type": "object", + "properties": { + "branch": { + "description": "Branch is the SCM branch to filter by.", + "type": "string" + }, + "end_time": { + "description": "EndTime is the end time for the time range filter.\nTime format is RFC3339: 2006-01-02T15:04:05Z07:00", + "type": "string" + }, + "labels": { + "description": "Labels filters SCM summaries by report labels.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "limit": { + "description": "Limit is the maximum number of SCMs to return.", + "type": "integer" + }, + "page": { + "description": "Page is the page number for pagination.", + "type": "integer" + }, + "scmid": { + "description": "ScmID is the ID of the SCM to filter by.", + "type": "string" + }, + "start_time": { + "description": "StartTime is the start time for the time range filter.\nTime format is RFC3339: 2006-01-02T15:04:05Z07:00", + "type": "string" + }, + "summary": { + "description": "Summary indicates if the response should contain SCM summary data.", + "type": "boolean" + }, + "url": { + "description": "URL is the SCM URL to filter by.", + "type": "string" + } + } + }, "server.SourceConfigResponse": { "type": "object", "properties": { diff --git a/docs/swagger.json b/docs/swagger.json index 3b7d0d95..97557645 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -430,6 +430,131 @@ } } }, + "/api/pipeline/labels": { + "get": { + "description": "List labels data from the database with optional filtering", + "tags": [ + "Labels" + ], + "summary": "List labels", + "parameters": [ + { + "type": "string", + "description": "Filter by label ID", + "name": "id", + "in": "query" + }, + { + "type": "string", + "description": "Filter by label key", + "name": "key", + "in": "query" + }, + { + "type": "string", + "description": "Filter by label value", + "name": "value", + "in": "query" + }, + { + "type": "string", + "description": "Return only unique label keys (true/false)", + "name": "keyonly", + "in": "query" + }, + { + "type": "string", + "description": "Limit the number of labels returned, default is 100", + "name": "limit", + "in": "query" + }, + { + "type": "string", + "description": "Page number for pagination, default is 1", + "name": "page", + "in": "query" + }, + { + "type": "string", + "description": "Start time for filtering labels (RFC3339 format)", + "name": "start_time", + "in": "query" + }, + { + "type": "string", + "description": "End time for filtering labels (RFC3339 format)", + "name": "end_time", + "in": "query" + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ListLabelsResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + } + } + } + }, + "/api/pipeline/labels/search": { + "post": { + "description": "Search labels in the database using advanced filtering", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Labels" + ], + "summary": "Search labels", + "parameters": [ + { + "description": "Search parameters", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/server.SearchLabelsRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ListLabelsResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + } + } + } + }, "/api/pipeline/reports": { "get": { "description": "List all pipeline reports from the database", @@ -509,6 +634,12 @@ "$ref": "#/definitions/server.CreatePipelineReportResponse" } }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + }, "500": { "description": "Internal Server Error", "schema": { @@ -567,6 +698,52 @@ } } }, + "/api/pipeline/reports/summary": { + "post": { + "description": "Return the number of pipeline reports per result for each time bucket of the requested time range.\nBuckets are UTC hours, UTC calendar days, ISO weeks or calendar months depending on the\ngranularity, and the date of an entry is the start of its bucket, formatted as RFC3339.\nEvery report is counted, including several reports of the same pipeline, and buckets without\nany report are returned with a zeroed entry.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "Pipeline Reports" + ], + "summary": "Summarize pipeline reports", + "parameters": [ + { + "description": "Summary filters", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/server.SearchPipelineReportsSummaryRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.SearchPipelineReportsSummaryResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + } + } + } + }, "/api/pipeline/reports/{id}": { "get": { "description": "Get the latest pipeline report for a specific ID", @@ -584,8 +761,8 @@ } ], "responses": { - "201": { - "description": "Created", + "200": { + "description": "OK", "schema": { "$ref": "#/definitions/server.GetPipelineReportByIDResponse" } @@ -650,8 +827,8 @@ } ], "responses": { - "201": { - "description": "Created", + "200": { + "description": "OK", "schema": { "$ref": "#/definitions/server.DefaultResponseModel" } @@ -725,6 +902,58 @@ "responses": { "200": { "description": "OK", + "schema": { + "$ref": "#/definitions/server.ListSCMsResponse" + } + }, + "400": { + "description": "Bad Request", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + }, + "500": { + "description": "Internal Server Error", + "schema": { + "$ref": "#/definitions/server.DefaultResponseModel" + } + } + } + } + }, + "/api/pipeline/scms/search": { + "post": { + "description": "Search SCM data using JSON filters. When summary is true, the response contains SCM summary data for all matching SCMs.", + "consumes": [ + "application/json" + ], + "produces": [ + "application/json" + ], + "tags": [ + "SCMs" + ], + "summary": "Search SCMs", + "parameters": [ + { + "description": "SCM search filters", + "name": "body", + "in": "body", + "required": true, + "schema": { + "$ref": "#/definitions/server.SearchSCMsRequest" + } + } + ], + "responses": { + "200": { + "description": "OK", + "schema": { + "$ref": "#/definitions/server.ListSCMsResponse" + } + }, + "400": { + "description": "Bad Request", "schema": { "$ref": "#/definitions/server.DefaultResponseModel" } @@ -801,9 +1030,37 @@ } } }, + "database.ReportResultSummaryEntry": { + "type": "object", + "properties": { + "date": { + "description": "Date is the start of the bucket, in UTC, formatted as RFC3339.", + "type": "string" + }, + "results": { + "description": "Results contains the number of reports per Updatecli result for that bucket.", + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, + "total": { + "description": "Total is the number of reports for that bucket, all results combined.", + "type": "integer" + } + } + }, "database.SearchLatestReportData": { "type": "object", "properties": { + "conditionConfigIDs": { + "description": "ConditionConfigIDs contains the config condition IDs associated with the report.", + "allOf": [ + { + "$ref": "#/definitions/pgtype.Hstore" + } + ] + }, "createdAt": { "description": "CreatedAt represents the creation date of the report.", "type": "string" @@ -832,6 +1089,22 @@ "description": "Result represents the result of the report.", "type": "string" }, + "sourceConfigIDs": { + "description": "SourceConfigIDs contains the config source IDs associated with the report.", + "allOf": [ + { + "$ref": "#/definitions/pgtype.Hstore" + } + ] + }, + "targetConfigIDs": { + "description": "TargetConfigIDs contains the config target IDs associated with the report.", + "allOf": [ + { + "$ref": "#/definitions/pgtype.Hstore" + } + ] + }, "updatedAt": { "description": "UpdatedAt represents the last update date of the report.", "type": "string" @@ -915,15 +1188,45 @@ } } }, + "model.Label": { + "type": "object", + "properties": { + "created_at": { + "description": "CreatedAt is the time the label was created", + "type": "string" + }, + "id": { + "description": "ID is a unique identifier for the label, generated as a UUID.", + "type": "string" + }, + "key": { + "description": "Key is the label name", + "type": "string" + }, + "last_pipeline_report_at": { + "description": "LastPipelineReportAt is the time the label was last used in a pipeline report", + "type": "string" + }, + "updated_at": { + "description": "UpdatedAt is the time the label was last updated", + "type": "string" + }, + "value": { + "description": "Value is the value associated with the label", + "type": "string" + } + } + }, "model.PipelineReport": { "type": "object", "properties": { "conditionConfigIDs": { "description": "ConditionConfigIDs is a list of unique identifiers of the condition configuration associated with the database.", - "type": "object", - "additionalProperties": { - "type": "string" - } + "allOf": [ + { + "$ref": "#/definitions/pgtype.Hstore" + } + ] }, "created_at": { "description": "Create_at represent the creation date of the record.", @@ -933,6 +1236,13 @@ "description": "ID is the unique identifier of the record in the database.", "type": "string" }, + "labelIDs": { + "description": "LabelIDs is a list of unique identifiers of the labels associated with the database.", + "type": "array", + "items": { + "type": "string" + } + }, "pipeline": { "description": "Pipeline represent the Updatecli pipeline report.", "allOf": [ @@ -955,17 +1265,19 @@ }, "sourceConfigIDs": { "description": "SourceConfigIDs is a list of unique identifiers of the source configuration associated with the database.", - "type": "object", - "additionalProperties": { - "type": "string" - } + "allOf": [ + { + "$ref": "#/definitions/pgtype.Hstore" + } + ] }, "targetConfigIDs": { "description": "TargetConfigIDs is a list of unique identifiers of the target configuration associated with the database.", - "type": "object", - "additionalProperties": { - "type": "string" - } + "allOf": [ + { + "$ref": "#/definitions/pgtype.Hstore" + } + ] }, "targetScmIDs": { "description": "TargetScmIDs is a list of unique identifiers of the scm configuration associated with the database.", @@ -980,6 +1292,37 @@ } } }, + "model.SCM": { + "type": "object", + "properties": { + "branch": { + "description": "Branch is the Git branch", + "type": "string" + }, + "created_at": { + "description": "Created_at is the time the SCM configuration was created", + "type": "string" + }, + "id": { + "description": "ID is a unique identifier for the SCM configuration", + "type": "string" + }, + "updated_at": { + "description": "Updated_at is the time the SCM configuration was last updated", + "type": "string" + }, + "url": { + "description": "URL is the Git repository URL", + "type": "string" + } + } + }, + "pgtype.Hstore": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, "reports.Action": { "type": "object", "properties": { @@ -1053,6 +1396,17 @@ } } }, + "reports.CIData": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "url": { + "type": "string" + } + } + }, "reports.PipelineURL": { "type": "object", "properties": { @@ -1075,6 +1429,9 @@ "$ref": "#/definitions/reports.Action" } }, + "ci": { + "$ref": "#/definitions/reports.CIData" + }, "conditions": { "type": "object", "additionalProperties": { @@ -1091,6 +1448,12 @@ "description": "ID defines the report ID", "type": "string" }, + "labels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, "name": { "type": "string" }, @@ -1434,6 +1797,183 @@ } } }, + "server.ListLabelsResponse": { + "type": "object", + "properties": { + "labels": { + "description": "Labels is a list of labels.", + "type": "array", + "items": { + "$ref": "#/definitions/model.Label" + } + }, + "total_count": { + "description": "TotalCount is the total number of labels matching the query.", + "type": "integer" + } + } + }, + "server.ListSCMsResponse": { + "type": "object", + "properties": { + "scms": { + "description": "SCMs is a list of SCMs.", + "type": "array", + "items": { + "$ref": "#/definitions/model.SCM" + } + }, + "total_count": { + "description": "TotalCount is the total number of SCMs matching the query.", + "type": "integer" + } + } + }, + "server.SearchLabelsRequest": { + "type": "object", + "properties": { + "end_time": { + "description": "EndTime is the end time for the time range filter\nThis is optional and can be used to filter labels by a specific end time\nTime format is RFC3339: 2006-01-02T15:04:05Z07:00", + "type": "string" + }, + "id": { + "description": "Id is the unique identifier of the label.", + "type": "string" + }, + "key": { + "description": "Key is the key of the label.", + "type": "string" + }, + "key_only": { + "description": "KeyOnly specifies if we only need to retrieve a list of uniq Label keys", + "type": "boolean" + }, + "limit": { + "description": "Limit is the maximum number of labels to return\nThis is optional and can be used to limit the number of labels returned", + "type": "integer" + }, + "page": { + "description": "Page is the page number for pagination\nThis is optional and can be used to paginate the results", + "type": "integer" + }, + "start_time": { + "description": "StartTime is the start time for the time range filter\nThis is optional and can be used to filter labels by a specific start time\nTime format is RFC3339: 2006-01-02T15:04:05Z07:00", + "type": "string" + }, + "value": { + "description": "Value is the value of the label.", + "type": "string" + } + } + }, + "server.SearchPipelineReportsSummaryRequest": { + "type": "object", + "properties": { + "days": { + "description": "Days is the number of days to summarize, today included.\nIt defaults to 7 and is ignored when hours, or start_time and end_time, are provided.", + "type": "integer" + }, + "end_time": { + "description": "EndTime is the end time for the time range filter.\nTime format is: 2006-01-02 15:04:05Z07:00", + "type": "string" + }, + "granularity": { + "description": "Granularity is the size of the time buckets, one of \"hour\", \"day\", \"week\" or\n\"month\". It defaults to \"day\".", + "type": "string" + }, + "hours": { + "description": "Hours is the number of hours to summarize, the current hour included.\nIt cannot be combined with days and is ignored when start_time and end_time are provided.", + "type": "integer" + }, + "labels": { + "description": "Labels is a map of labels to filter reports by.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "metric": { + "description": "Metric is what the reports are counted by. It defaults to \"result\", which is\nthe only value supported so far.", + "type": "string" + }, + "scmid": { + "description": "ScmID is the ID of the SCM to filter reports by.\nUse \"none\" to only count the reports which are not attached to any SCM.", + "type": "string" + }, + "start_time": { + "description": "StartTime is the start time for the time range filter.\nTime format is: 2006-01-02 15:04:05Z07:00", + "type": "string" + } + } + }, + "server.SearchPipelineReportsSummaryResponse": { + "type": "object", + "properties": { + "data": { + "description": "Data contains one entry per time bucket, ordered from the oldest to the most recent one.", + "type": "array", + "items": { + "$ref": "#/definitions/database.ReportResultSummaryEntry" + } + }, + "granularity": { + "description": "Granularity is the size of the time buckets of the entries.", + "type": "string" + }, + "metric": { + "description": "Metric is the metric the reports were counted by.", + "type": "string" + }, + "total_count": { + "description": "TotalCount is the total number of reports matching the query.", + "type": "integer" + } + } + }, + "server.SearchSCMsRequest": { + "type": "object", + "properties": { + "branch": { + "description": "Branch is the SCM branch to filter by.", + "type": "string" + }, + "end_time": { + "description": "EndTime is the end time for the time range filter.\nTime format is RFC3339: 2006-01-02T15:04:05Z07:00", + "type": "string" + }, + "labels": { + "description": "Labels filters SCM summaries by report labels.", + "type": "object", + "additionalProperties": { + "type": "string" + } + }, + "limit": { + "description": "Limit is the maximum number of SCMs to return.", + "type": "integer" + }, + "page": { + "description": "Page is the page number for pagination.", + "type": "integer" + }, + "scmid": { + "description": "ScmID is the ID of the SCM to filter by.", + "type": "string" + }, + "start_time": { + "description": "StartTime is the start time for the time range filter.\nTime format is RFC3339: 2006-01-02T15:04:05Z07:00", + "type": "string" + }, + "summary": { + "description": "Summary indicates if the response should contain SCM summary data.", + "type": "boolean" + }, + "url": { + "description": "URL is the SCM URL to filter by.", + "type": "string" + } + } + }, "server.SourceConfigResponse": { "type": "object", "properties": { diff --git a/docs/swagger.yaml b/docs/swagger.yaml index f7a7221c..4cba9546 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -70,8 +70,28 @@ definitions: $ref: '#/definitions/transformer.Transformer' type: array type: object + database.ReportResultSummaryEntry: + properties: + date: + description: Date is the start of the bucket, in UTC, formatted as RFC3339. + type: string + results: + additionalProperties: + type: integer + description: Results contains the number of reports per Updatecli result for + that bucket. + type: object + total: + description: Total is the number of reports for that bucket, all results combined. + type: integer + type: object database.SearchLatestReportData: properties: + conditionConfigIDs: + allOf: + - $ref: '#/definitions/pgtype.Hstore' + description: ConditionConfigIDs contains the config condition IDs associated + with the report. createdAt: description: CreatedAt represents the creation date of the report. type: string @@ -93,6 +113,16 @@ definitions: result: description: Result represents the result of the report. type: string + sourceConfigIDs: + allOf: + - $ref: '#/definitions/pgtype.Hstore' + description: SourceConfigIDs contains the config source IDs associated with + the report. + targetConfigIDs: + allOf: + - $ref: '#/definitions/pgtype.Hstore' + description: TargetConfigIDs contains the config target IDs associated with + the report. updatedAt: description: UpdatedAt represents the last update date of the report. type: string @@ -150,20 +180,47 @@ definitions: description: Updated_at represent the last update date of the record. type: string type: object + model.Label: + properties: + created_at: + description: CreatedAt is the time the label was created + type: string + id: + description: ID is a unique identifier for the label, generated as a UUID. + type: string + key: + description: Key is the label name + type: string + last_pipeline_report_at: + description: LastPipelineReportAt is the time the label was last used in a + pipeline report + type: string + updated_at: + description: UpdatedAt is the time the label was last updated + type: string + value: + description: Value is the value associated with the label + type: string + type: object model.PipelineReport: properties: conditionConfigIDs: - additionalProperties: - type: string + allOf: + - $ref: '#/definitions/pgtype.Hstore' description: ConditionConfigIDs is a list of unique identifiers of the condition configuration associated with the database. - type: object created_at: description: Create_at represent the creation date of the record. type: string id: description: ID is the unique identifier of the record in the database. type: string + labelIDs: + description: LabelIDs is a list of unique identifiers of the labels associated + with the database. + items: + type: string + type: array pipeline: allOf: - $ref: '#/definitions/reports.Report' @@ -183,17 +240,15 @@ definitions: description: Result represent the result of the pipeline execution. type: string sourceConfigIDs: - additionalProperties: - type: string + allOf: + - $ref: '#/definitions/pgtype.Hstore' description: SourceConfigIDs is a list of unique identifiers of the source configuration associated with the database. - type: object targetConfigIDs: - additionalProperties: - type: string + allOf: + - $ref: '#/definitions/pgtype.Hstore' description: TargetConfigIDs is a list of unique identifiers of the target configuration associated with the database. - type: object targetScmIDs: description: TargetScmIDs is a list of unique identifiers of the scm configuration associated with the database. @@ -204,6 +259,28 @@ definitions: description: Updated_at represent the last update date of the record. type: string type: object + model.SCM: + properties: + branch: + description: Branch is the Git branch + type: string + created_at: + description: Created_at is the time the SCM configuration was created + type: string + id: + description: ID is a unique identifier for the SCM configuration + type: string + updated_at: + description: Updated_at is the time the SCM configuration was last updated + type: string + url: + description: URL is the Git repository URL + type: string + type: object + pgtype.Hstore: + additionalProperties: + type: string + type: object reports.Action: properties: actionUrl: @@ -253,6 +330,13 @@ definitions: description: Title is the title of the changelog type: string type: object + reports.CIData: + properties: + name: + type: string + url: + type: string + type: object reports.PipelineURL: properties: name: @@ -268,6 +352,8 @@ definitions: additionalProperties: $ref: '#/definitions/reports.Action' type: object + ci: + $ref: '#/definitions/reports.CIData' conditions: additionalProperties: $ref: '#/definitions/result.Condition' @@ -279,6 +365,10 @@ definitions: id: description: ID defines the report ID type: string + labels: + additionalProperties: + type: string + type: object name: type: string pipelineID: @@ -520,6 +610,164 @@ definitions: total_count: type: integer type: object + server.ListLabelsResponse: + properties: + labels: + description: Labels is a list of labels. + items: + $ref: '#/definitions/model.Label' + type: array + total_count: + description: TotalCount is the total number of labels matching the query. + type: integer + type: object + server.ListSCMsResponse: + properties: + scms: + description: SCMs is a list of SCMs. + items: + $ref: '#/definitions/model.SCM' + type: array + total_count: + description: TotalCount is the total number of SCMs matching the query. + type: integer + type: object + server.SearchLabelsRequest: + properties: + end_time: + description: |- + EndTime is the end time for the time range filter + This is optional and can be used to filter labels by a specific end time + Time format is RFC3339: 2006-01-02T15:04:05Z07:00 + type: string + id: + description: Id is the unique identifier of the label. + type: string + key: + description: Key is the key of the label. + type: string + key_only: + description: KeyOnly specifies if we only need to retrieve a list of uniq + Label keys + type: boolean + limit: + description: |- + Limit is the maximum number of labels to return + This is optional and can be used to limit the number of labels returned + type: integer + page: + description: |- + Page is the page number for pagination + This is optional and can be used to paginate the results + type: integer + start_time: + description: |- + StartTime is the start time for the time range filter + This is optional and can be used to filter labels by a specific start time + Time format is RFC3339: 2006-01-02T15:04:05Z07:00 + type: string + value: + description: Value is the value of the label. + type: string + type: object + server.SearchPipelineReportsSummaryRequest: + properties: + days: + description: |- + Days is the number of days to summarize, today included. + It defaults to 7 and is ignored when hours, or start_time and end_time, are provided. + type: integer + end_time: + description: |- + EndTime is the end time for the time range filter. + Time format is: 2006-01-02 15:04:05Z07:00 + type: string + granularity: + description: |- + Granularity is the size of the time buckets, one of "hour", "day", "week" or + "month". It defaults to "day". + type: string + hours: + description: |- + Hours is the number of hours to summarize, the current hour included. + It cannot be combined with days and is ignored when start_time and end_time are provided. + type: integer + labels: + additionalProperties: + type: string + description: Labels is a map of labels to filter reports by. + type: object + metric: + description: |- + Metric is what the reports are counted by. It defaults to "result", which is + the only value supported so far. + type: string + scmid: + description: |- + ScmID is the ID of the SCM to filter reports by. + Use "none" to only count the reports which are not attached to any SCM. + type: string + start_time: + description: |- + StartTime is the start time for the time range filter. + Time format is: 2006-01-02 15:04:05Z07:00 + type: string + type: object + server.SearchPipelineReportsSummaryResponse: + properties: + data: + description: Data contains one entry per time bucket, ordered from the oldest + to the most recent one. + items: + $ref: '#/definitions/database.ReportResultSummaryEntry' + type: array + granularity: + description: Granularity is the size of the time buckets of the entries. + type: string + metric: + description: Metric is the metric the reports were counted by. + type: string + total_count: + description: TotalCount is the total number of reports matching the query. + type: integer + type: object + server.SearchSCMsRequest: + properties: + branch: + description: Branch is the SCM branch to filter by. + type: string + end_time: + description: |- + EndTime is the end time for the time range filter. + Time format is RFC3339: 2006-01-02T15:04:05Z07:00 + type: string + labels: + additionalProperties: + type: string + description: Labels filters SCM summaries by report labels. + type: object + limit: + description: Limit is the maximum number of SCMs to return. + type: integer + page: + description: Page is the page number for pagination. + type: integer + scmid: + description: ScmID is the ID of the SCM to filter by. + type: string + start_time: + description: |- + StartTime is the start time for the time range filter. + Time format is RFC3339: 2006-01-02T15:04:05Z07:00 + type: string + summary: + description: Summary indicates if the response should contain SCM summary + data. + type: boolean + url: + description: URL is the SCM URL to filter by. + type: string + type: object server.SourceConfigResponse: properties: configs: @@ -1083,6 +1331,88 @@ paths: summary: Search configuration targets tags: - Configuration Targets + /api/pipeline/labels: + get: + description: List labels data from the database with optional filtering + parameters: + - description: Filter by label ID + in: query + name: id + type: string + - description: Filter by label key + in: query + name: key + type: string + - description: Filter by label value + in: query + name: value + type: string + - description: Return only unique label keys (true/false) + in: query + name: keyonly + type: string + - description: Limit the number of labels returned, default is 100 + in: query + name: limit + type: string + - description: Page number for pagination, default is 1 + in: query + name: page + type: string + - description: Start time for filtering labels (RFC3339 format) + in: query + name: start_time + type: string + - description: End time for filtering labels (RFC3339 format) + in: query + name: end_time + type: string + responses: + "200": + description: OK + schema: + $ref: '#/definitions/server.ListLabelsResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/server.DefaultResponseModel' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/server.DefaultResponseModel' + summary: List labels + tags: + - Labels + /api/pipeline/labels/search: + post: + consumes: + - application/json + description: Search labels in the database using advanced filtering + parameters: + - description: Search parameters + in: body + name: body + required: true + schema: + $ref: '#/definitions/server.SearchLabelsRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/server.ListLabelsResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/server.DefaultResponseModel' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/server.DefaultResponseModel' + summary: Search labels + tags: + - Labels /api/pipeline/reports: get: consumes: @@ -1134,6 +1464,10 @@ paths: description: Created schema: $ref: '#/definitions/server.CreatePipelineReportResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/server.DefaultResponseModel' "500": description: Internal Server Error schema: @@ -1151,8 +1485,8 @@ paths: required: true type: string responses: - "201": - description: Created + "200": + description: OK schema: $ref: '#/definitions/server.DefaultResponseModel' "500": @@ -1171,8 +1505,8 @@ paths: required: true type: string responses: - "201": - description: Created + "200": + description: OK schema: $ref: '#/definitions/server.GetPipelineReportByIDResponse' "404": @@ -1239,6 +1573,41 @@ paths: summary: Search pipeline reports tags: - Pipeline Reports + /api/pipeline/reports/summary: + post: + consumes: + - application/json + description: |- + Return the number of pipeline reports per result for each time bucket of the requested time range. + Buckets are UTC hours, UTC calendar days, ISO weeks or calendar months depending on the + granularity, and the date of an entry is the start of its bucket, formatted as RFC3339. + Every report is counted, including several reports of the same pipeline, and buckets without + any report are returned with a zeroed entry. + parameters: + - description: Summary filters + in: body + name: body + required: true + schema: + $ref: '#/definitions/server.SearchPipelineReportsSummaryRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/server.SearchPipelineReportsSummaryResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/server.DefaultResponseModel' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/server.DefaultResponseModel' + summary: Summarize pipeline reports + tags: + - Pipeline Reports /api/pipeline/scms: get: description: List SCMs data from the database @@ -1278,6 +1647,10 @@ paths: responses: "200": description: OK + schema: + $ref: '#/definitions/server.ListSCMsResponse' + "400": + description: Bad Request schema: $ref: '#/definitions/server.DefaultResponseModel' "500": @@ -1287,4 +1660,35 @@ paths: summary: List SCMs tags: - SCMs + /api/pipeline/scms/search: + post: + consumes: + - application/json + description: Search SCM data using JSON filters. When summary is true, the response + contains SCM summary data for all matching SCMs. + parameters: + - description: SCM search filters + in: body + name: body + required: true + schema: + $ref: '#/definitions/server.SearchSCMsRequest' + produces: + - application/json + responses: + "200": + description: OK + schema: + $ref: '#/definitions/server.ListSCMsResponse' + "400": + description: Bad Request + schema: + $ref: '#/definitions/server.DefaultResponseModel' + "500": + description: Internal Server Error + schema: + $ref: '#/definitions/server.DefaultResponseModel' + summary: Search SCMs + tags: + - SCMs swagger: "2.0" diff --git a/pkg/database/database_test.go b/pkg/database/database_test.go index 296a2589..3fc814fa 100644 --- a/pkg/database/database_test.go +++ b/pkg/database/database_test.go @@ -3,10 +3,13 @@ package database import ( "context" "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/updatecli/udash/test" + "github.com/updatecli/updatecli/pkg/core/reports" + "github.com/updatecli/updatecli/pkg/core/result" ) func TestDatabase(t *testing.T) { @@ -25,4 +28,77 @@ func TestDatabase(t *testing.T) { t.Log("Postgres Container connected") require.NoError(t, RunMigrationUp()) t.Log("Postgres Container migrations run") + + t.Run("truncateToBucket matches date_trunc", func(t *testing.T) { + // The summary zero fills its buckets from truncateToBucket while the counted + // rows are bucketed by date_trunc. Any divergence between the two silently + // drops reports from the dataset, so they are compared here rather than left + // to the endpoint tests to notice. + granularities := []SummaryGranularity{ + SummaryGranularityHour, + SummaryGranularityDay, + SummaryGranularityWeek, + SummaryGranularityMonth, + } + + // A monday, a sunday, the first and the last day of a month, a leap day and + // the boundaries of a day. + samples := []time.Time{ + time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC), + time.Date(2026, 1, 5, 13, 45, 12, 0, time.UTC), + time.Date(2026, 1, 11, 23, 59, 59, 0, time.UTC), + time.Date(2026, 2, 28, 12, 0, 0, 0, time.UTC), + time.Date(2024, 2, 29, 6, 30, 0, 0, time.UTC), + time.Date(2026, 12, 31, 23, 0, 0, 0, time.UTC), + } + + for _, granularity := range granularities { + for _, sample := range samples { + want := time.Time{} + require.NoError(t, DB.QueryRow(ctx, + "SELECT date_trunc($1, $2::timestamp)", string(granularity), sample, + ).Scan(&want)) + + assert.Equal(t, want.UTC(), truncateToBucket(sample, granularity), + "granularity %q, sample %s", granularity, sample) + } + } + }) + + t.Run("migration 000010 backfills pipeline_result", func(t *testing.T) { + // Migration 000004 read "data ->> 'result'" while a marshalled report stores + // the key as "Result", so its backfill silently did nothing and every report + // inserted before it still has an empty pipeline_result. + id, err := InsertReport(ctx, reports.Report{ + Name: "ci: bump Venom version", + Result: result.SUCCESS, + ID: "1de1797bbc925e08e473178425b11eb16fc547291f4b45274da24c2b00e2afc3", + PipelineID: "venom", + }) + require.NoError(t, err) + t.Cleanup(func() { + _, err := DB.Exec(ctx, "DELETE FROM pipelineReports WHERE id = $1", id) + assert.NoError(t, err) + }) + + _, err = DB.Exec(ctx, + "UPDATE pipelineReports SET pipeline_result = '', pipeline_name = '' WHERE id = $1", id) + require.NoError(t, err) + + // Replaying the migration itself rather than a copy of its statements is what + // makes this a regression test for the jsonb key casing. + migration, err := fs.ReadFile("migrations/000010_fix_pipelineReports_denormalized_columns.up.sql") + require.NoError(t, err) + + _, err = DB.Exec(ctx, string(migration)) + require.NoError(t, err) + + pipelineResult, pipelineName := "", "" + require.NoError(t, DB.QueryRow(ctx, + "SELECT pipeline_result, pipeline_name FROM pipelineReports WHERE id = $1", id, + ).Scan(&pipelineResult, &pipelineName)) + + assert.Equal(t, result.SUCCESS, pipelineResult) + assert.Equal(t, "ci: bump Venom version", pipelineName) + }) } diff --git a/pkg/database/migrations/000010_fix_pipelineReports_denormalized_columns.down.sql b/pkg/database/migrations/000010_fix_pipelineReports_denormalized_columns.down.sql new file mode 100644 index 00000000..6d674f31 --- /dev/null +++ b/pkg/database/migrations/000010_fix_pipelineReports_denormalized_columns.down.sql @@ -0,0 +1,7 @@ +-- Only the index is dropped: emptying pipeline_result and pipeline_name again would +-- destroy data rather than restore the previous state. +BEGIN; + +DROP INDEX IF EXISTS idx_pipelinereports_updated_at_pipeline_result; + +COMMIT; diff --git a/pkg/database/migrations/000010_fix_pipelineReports_denormalized_columns.up.sql b/pkg/database/migrations/000010_fix_pipelineReports_denormalized_columns.up.sql new file mode 100644 index 00000000..ac93eeec --- /dev/null +++ b/pkg/database/migrations/000010_fix_pipelineReports_denormalized_columns.up.sql @@ -0,0 +1,24 @@ +-- Migration 000004 backfilled pipeline_result and pipeline_name from "data ->> 'result'" +-- and "data ->> 'name'", but a marshalled report stores those keys as "Result" and "Name". +-- jsonb keys are case sensitive so NULLIF(TRIM(...), '') always evaluated to NULL, the +-- COALESCE fell back to the column's own default and the backfill did nothing. Only +-- pipeline_id used the right casing, which is why it is the only one of the three that is +-- queried today. Every report inserted before 000004 therefore still has an empty +-- pipeline_result, which the reports summary would report as an unknown result. +BEGIN; + +UPDATE pipelineReports +SET + pipeline_result = COALESCE(NULLIF(TRIM(data ->> 'Result'), ''), pipeline_result), + pipeline_name = COALESCE(NULLIF(TRIM(data ->> 'Name'), ''), pipeline_name) +WHERE + TRIM(pipeline_result) = '' + OR TRIM(pipeline_name) = ''; + +-- The reports summary groups the reports of a time range per result. idx_pipelinereports_updated_at +-- already serves the range predicate but the result still has to be fetched from the heap +-- row by row, so a composite index is what makes the aggregation an index only scan. +CREATE INDEX IF NOT EXISTS idx_pipelinereports_updated_at_pipeline_result +ON pipelineReports (updated_at, pipeline_result); + +COMMIT; diff --git a/pkg/database/report.go b/pkg/database/report.go index b88a2d1d..bbe44528 100644 --- a/pkg/database/report.go +++ b/pkg/database/report.go @@ -3,8 +3,10 @@ package database import ( "context" "encoding/json" + "errors" "fmt" "slices" + "time" "github.com/google/uuid" "github.com/jackc/pgx/v5/pgtype" @@ -17,6 +19,7 @@ import ( "github.com/stephenafamo/bob/dialect/psql/sm" "github.com/updatecli/udash/pkg/model" "github.com/updatecli/updatecli/pkg/core/reports" + "github.com/updatecli/updatecli/pkg/core/result" ) // SearchLatestReportData represents a report. @@ -99,8 +102,6 @@ type SearchLatestReportsParams struct { } // SearchLatestReports searches the latest reports according some parameters. -// -//nolint:funlen func SearchLatestReports(params SearchLatestReportsParams) ([]SearchLatestReportData, int, error) { queryString := "" var args []any @@ -167,39 +168,8 @@ func SearchLatestReports(params SearchLatestReportsParams) ([]SearchLatestReport } } - switch params.ScmID { - case "": - case "none", "null", "nil": - query.Apply( - sm.Where( - psql.Or( - psql.Quote("cardinality(target_db_scm_ids) = 0"), - psql.Quote("target_db_scm_ids").IsNull(), - ), - ), - ) - - default: - scm, _, err := GetSCM(params.Ctx, params.ScmID, "", "", 0, 1) - if err != nil { - logrus.Errorf("get scm data: %s", err) - return nil, 0, err - } - - switch len(scm) { - case 0: - logrus.Errorf("scm data not found") - case 1: - query.Apply( - sm.Where( - psql.Raw(`target_db_scm_ids && ?`, fmt.Sprintf("{%s}", scm[0].ID.String())), - ), - ) - default: - // Normally we should never have multiple scms with the same id - // so we should never reach this point. - logrus.Errorf("unexpected behavior: multiple scms found") - } + if err := applyScmFilter(params.Ctx, &query, params.ScmID); err != nil { + return nil, 0, err } // Total counter query must be built before applying pagination @@ -318,6 +288,323 @@ func SearchLatestReports(params SearchLatestReportsParams) ([]SearchLatestReport return dataset, totalCount, nil } +// SummaryGranularity is the size of the time buckets a reports summary is grouped by. +type SummaryGranularity string + +const ( + // SummaryGranularityHour groups the reports per UTC hour. + SummaryGranularityHour SummaryGranularity = "hour" + // SummaryGranularityDay groups the reports per UTC day. + SummaryGranularityDay SummaryGranularity = "day" + // SummaryGranularityWeek groups the reports per ISO week, starting on monday. + SummaryGranularityWeek SummaryGranularity = "week" + // SummaryGranularityMonth groups the reports per calendar month. + SummaryGranularityMonth SummaryGranularity = "month" +) + +// IsValid reports whether the granularity is one this package knows how to bucket. +func (g SummaryGranularity) IsValid() bool { + switch g { + case SummaryGranularityHour, SummaryGranularityDay, SummaryGranularityWeek, SummaryGranularityMonth: + return true + default: + return false + } +} + +// ErrSummaryRangeTooWide is returned when the requested time range spans more days than +// the caller allows. Callers are expected to turn it into a client error. +var ErrSummaryRangeTooWide = errors.New("requested time range is too wide") + +// ErrSummaryTooManyBuckets is returned when the requested time range and granularity would +// produce more buckets than the caller allows. Callers are expected to turn it into a +// client error. +var ErrSummaryTooManyBuckets = errors.New("requested time range produces too many buckets") + +// summaryDateFormat is the layout used to identify the bucket of a summary entry. It has to +// carry the time of the day, otherwise every bucket of an hourly summary would share the +// same identifier and their counts would be merged together. +const summaryDateFormat = time.RFC3339 + +// summaryUnknownResult is the key reporting the reports whose result is empty or is not +// an Updatecli result. +const summaryUnknownResult = "unknown" + +// summaryResultKeys contains the keys always reported for a bucket, even when no report +// matched, so that consumers always retrieve the same set of keys. +var summaryResultKeys = []string{ + result.SUCCESS, + result.FAILURE, + result.ATTENTION, + result.SKIPPED, + summaryUnknownResult, +} + +// ReportSummaryParams contains the parameters used to summarize reports per time bucket. +type ReportSummaryParams struct { + Ctx context.Context + // Days is how far back to look for reports, in days. + // It is ignored when Hours, or StartTime and EndTime, are provided. + Days int + // Hours is how far back to look for reports, in hours. It takes precedence over + // Days and is ignored when StartTime and EndTime are provided. + Hours int + // Granularity is the size of the time buckets, it defaults to a day. + Granularity SummaryGranularity + // MaxDays is the widest time range accepted, in days. A value lower than one + // does not enforce any limit. + MaxDays int + // MaxBuckets is the largest number of buckets a summary may return. A value lower + // than one does not enforce any limit. + MaxBuckets int + // StartTime and EndTime define an explicit time range, both must be provided. + StartTime string + EndTime string + // ScmID restricts the summary to the reports of a specific scm. + ScmID string + // Labels restricts the summary to the reports matching those labels. + Labels map[string]string +} + +// ReportResultSummaryEntry contains the number of reports per result for a single time bucket. +type ReportResultSummaryEntry struct { + // Date is the start of the bucket, in UTC, formatted as RFC3339. + Date string `json:"date"` + // Results contains the number of reports per Updatecli result for that bucket. + Results map[string]int `json:"results"` + // Total is the number of reports for that bucket, all results combined. + Total int `json:"total"` +} + +// SearchReportsSummary returns the number of reports per result for each time bucket of +// the requested time range. Buckets without any report are reported with a zeroed entry +// so that the returned dataset always covers the whole time range. +// +// The summary always covers whole buckets: an explicit time range is widened to the +// buckets it overlaps, otherwise a partial bucket would be reported as a drop of activity. +func SearchReportsSummary(params ReportSummaryParams) ([]ReportResultSummaryEntry, int, error) { + + granularity := params.Granularity + if granularity == "" { + granularity = SummaryGranularityDay + } + + if !granularity.IsValid() { + return nil, 0, fmt.Errorf("unsupported granularity %q", params.Granularity) + } + + firstBucket, lastBucket, err := summaryRange(params, granularity) + if err != nil { + return nil, 0, fmt.Errorf("resolving summary range: %w", err) + } + + // granularity is one of the constants above, never the raw value received from a + // caller, so it cannot inject anything into the query. + dateTrunc := fmt.Sprintf("date_trunc('%s', updated_at)", granularity) + + query := psql.Select( + sm.From("pipelineReports"), + sm.Columns( + dateTrunc, + // pipeline_result is denormalized from data ->> 'Result' when the report is + // inserted, grouping on it avoids parsing the jsonb document of every report. + "pipeline_result", + "count(*)", + ), + sm.Where( + psql.Raw("updated_at >= ? AND updated_at < ?", firstBucket, nextBucket(lastBucket, granularity)), + ), + sm.GroupBy(dateTrunc), + sm.GroupBy("pipeline_result"), + sm.OrderBy(dateTrunc), + ) + + if err := applyScmFilter(params.Ctx, &query, params.ScmID); err != nil { + return nil, 0, err + } + + if len(params.Labels) > 0 { + // The report window is widened to whole buckets so the label lookup must cover + // the same range, otherwise labels timestamped within the widened part would be + // missed and their reports silently dropped. An empty range keeps the lookup + // unbounded, as SearchLatestReports does. + labelStartTime, labelEndTime := "", "" + if params.StartTime != "" || params.EndTime != "" { + labelStartTime = firstBucket.Format(timeRangeLayout) + labelEndTime = nextBucket(lastBucket, granularity).Format(timeRangeLayout) + } + + if err := applyLabelFilter(labelFilterParams{ + Ctx: params.Ctx, + Query: &query, + Labels: params.Labels, + StartTime: labelStartTime, + EndTime: labelEndTime, + }); err != nil { + return nil, 0, fmt.Errorf("applying label filter: %w", err) + } + } + + queryString, args, err := query.Build(params.Ctx) + if err != nil { + return nil, 0, fmt.Errorf("building query failed: %s\n\t%s", queryString, err) + } + + rows, err := DB.Query(params.Ctx, queryString, args...) + if err != nil { + return nil, 0, fmt.Errorf("query failed: %q\n\t%s", queryString, err) + } + defer rows.Close() + + countByDate := map[string]map[string]int{} + totalCount := 0 + + for rows.Next() { + bucket := time.Time{} + reportResult := "" + count := 0 + + if err := rows.Scan(&bucket, &reportResult, &count); err != nil { + return nil, 0, fmt.Errorf("parsing result: %s", err) + } + + date := bucket.UTC().Format(summaryDateFormat) + if countByDate[date] == nil { + countByDate[date] = map[string]int{} + } + + countByDate[date][summaryResultKey(reportResult)] += count + totalCount += count + } + + if err := rows.Err(); err != nil { + return nil, 0, fmt.Errorf("reading results: %s", err) + } + + dataset := []ReportResultSummaryEntry{} + for bucket := firstBucket; !bucket.After(lastBucket); bucket = nextBucket(bucket, granularity) { + entry := ReportResultSummaryEntry{ + Date: bucket.Format(summaryDateFormat), + Results: map[string]int{}, + } + + for _, r := range summaryResultKeys { + entry.Results[r] = 0 + } + + for r, count := range countByDate[entry.Date] { + entry.Results[r] += count + entry.Total += count + } + + dataset = append(dataset, entry) + } + + return dataset, totalCount, nil +} + +// summaryResultKey maps a stored pipeline result to the key it is reported under. +// Anything unexpected, including the empty result of a report inserted before the +// pipeline_result column was backfilled, is folded into a single bucket so that the +// reported keys stay stable. +func summaryResultKey(pipelineResult string) string { + switch pipelineResult { + case result.SUCCESS, result.FAILURE, result.ATTENTION, result.SKIPPED: + return pipelineResult + default: + return summaryUnknownResult + } +} + +// summaryRange returns the first and the last bucket, both included, covered by a +// summary. Both are the start of a bucket, in UTC. +func summaryRange(params ReportSummaryParams, granularity SummaryGranularity) (time.Time, time.Time, error) { + + firstTime, lastTime := time.Time{}, time.Time{} + + switch { + case params.StartTime != "" || params.EndTime != "": + var err error + firstTime, lastTime, err = resolveTimeRange(0, params.StartTime, params.EndTime) + if err != nil { + return time.Time{}, time.Time{}, err + } + + case params.Hours > 0: + // The window includes the bucket of the current hour, as the Days one includes + // the bucket of the current day. + lastTime = time.Now().UTC() + firstTime = lastTime.Add(-time.Duration(params.Hours-1) * time.Hour) + + default: + days := params.Days + if days < 1 { + days = 1 + } + + lastTime = time.Now().UTC() + firstTime = lastTime.AddDate(0, 0, -(days - 1)) + } + + // The limit is checked against the requested range rather than the widened one: + // widening adds up to a bucket on each side, which a month granularity would + // otherwise turn into a rejection of a request that is within the limit. + if params.MaxDays > 0 && lastTime.Sub(firstTime) > time.Duration(params.MaxDays)*24*time.Hour { + return time.Time{}, time.Time{}, ErrSummaryRangeTooWide + } + + firstBucket := truncateToBucket(firstTime, granularity) + lastBucket := truncateToBucket(lastTime, granularity) + + // MaxDays bounds how much of the table the query scans, this bounds how large the + // response gets: an hourly summary of a year is a cheap scan but ~8800 entries. + if params.MaxBuckets > 0 { + count := 0 + for bucket := firstBucket; !bucket.After(lastBucket); bucket = nextBucket(bucket, granularity) { + count++ + if count > params.MaxBuckets { + return time.Time{}, time.Time{}, ErrSummaryTooManyBuckets + } + } + } + + return firstBucket, lastBucket, nil +} + +// truncateToBucket returns the start, in UTC, of the bucket containing the provided time. +// It must return the same instant as the matching date_trunc call, otherwise the zeroed +// buckets would not line up with the counted rows. +func truncateToBucket(t time.Time, granularity SummaryGranularity) time.Time { + t = t.UTC() + + switch granularity { + case SummaryGranularityHour: + return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, time.UTC) + case SummaryGranularityWeek: + day := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) + // date_trunc truncates a week to its ISO monday. + return day.AddDate(0, 0, -((int(day.Weekday()) + 6) % 7)) + case SummaryGranularityMonth: + return time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, time.UTC) + default: + return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) + } +} + +// nextBucket returns the start of the bucket following the provided bucket start. +func nextBucket(t time.Time, granularity SummaryGranularity) time.Time { + switch granularity { + case SummaryGranularityHour: + return t.Add(time.Hour) + case SummaryGranularityWeek: + return t.AddDate(0, 0, 7) + case SummaryGranularityMonth: + return t.AddDate(0, 1, 0) + default: + return t.AddDate(0, 0, 1) + } +} + // InsertReport inserts a new report into the database. func InsertReport(ctx context.Context, report reports.Report) (string, error) { var err error @@ -678,3 +965,48 @@ func applyResourceConfigFilter(query *bob.BaseQuery[*dialect.SelectQuery], id, k ) return nil } + +// applyScmFilter restricts the given query to the reports associated to a specific scm. +// An empty scmID does not filter anything while "none", "null", or "nil" only keeps +// the reports which are not associated to any scm. +func applyScmFilter(ctx context.Context, query *bob.BaseQuery[*dialect.SelectQuery], scmID string) error { + + switch scmID { + case "": + case "none", "null", "nil": + // psql.Quote would quote the whole expression as a column identifier, + // so the cardinality call must be passed as a raw expression. + query.Apply( + sm.Where( + psql.Or( + psql.Raw("cardinality(target_db_scm_ids) = 0"), + psql.Quote("target_db_scm_ids").IsNull(), + ), + ), + ) + + default: + scm, _, err := GetSCM(ctx, scmID, "", "", 0, 1) + if err != nil { + logrus.Errorf("get scm data: %s", err) + return err + } + + switch len(scm) { + case 0: + logrus.Errorf("scm data not found") + case 1: + query.Apply( + sm.Where( + psql.Raw(`target_db_scm_ids && ?`, fmt.Sprintf("{%s}", scm[0].ID.String())), + ), + ) + default: + // Normally we should never have multiple scms with the same id + // so we should never reach this point. + logrus.Errorf("unexpected behavior: multiple scms found") + } + } + + return nil +} diff --git a/pkg/database/time_utils.go b/pkg/database/time_utils.go index dd45e146..2a174dee 100644 --- a/pkg/database/time_utils.go +++ b/pkg/database/time_utils.go @@ -10,6 +10,9 @@ import ( "github.com/stephenafamo/bob/dialect/psql/sm" ) +// timeRangeLayout is the layout used to parse the startTime and endTime filters. +const timeRangeLayout = "2006-01-02 15:04:05Z07:00" + // dateRangeFilterParams holds parameters for applying a date range filter to a query. type dateRangeFilterParams struct { Query *bob.BaseQuery[*dialect.SelectQuery] @@ -18,36 +21,32 @@ type dateRangeFilterParams struct { EndTime string } -// applyRangeFilter applies a time range filter to the given query based on the provided -// startTime and endTime strings in RFC3339 format. If both are empty and dateRangeDays is greater than zero, -// it filters records updated within the last dateRangeDays days. -func applyRangeFilter(columnName string, r dateRangeFilterParams) error { +// resolveTimeRange returns the time window, in UTC, described by the provided +// startTime and endTime strings. If both are empty and days is greater than zero, +// the window ends now and starts days days ago. If both are empty and days is not +// greater than zero, both returned times are zero, meaning that no time boundary applies. +func resolveTimeRange(days int, startTime, endTime string) (time.Time, time.Time, error) { - if r.StartTime == "" && r.EndTime == "" && r.DateRangeDays > 0 { - start := time.Now().UTC().Add(-time.Duration(r.DateRangeDays) * 24 * time.Hour) - r.Query.Apply( - sm.Where( - psql.Raw(columnName+" > ?", start), - ), - ) - return nil - } + if startTime == "" && endTime == "" { + if days <= 0 { + return time.Time{}, time.Time{}, nil + } - if r.StartTime == "" && r.EndTime == "" { - return nil + end := time.Now().UTC() + return end.Add(-time.Duration(days) * 24 * time.Hour), end, nil } - if r.StartTime == "" || r.EndTime == "" { - return fmt.Errorf("both startTime %q and endTime %q must be provided for time range filtering", r.StartTime, r.EndTime) + if startTime == "" || endTime == "" { + return time.Time{}, time.Time{}, fmt.Errorf("both startTime %q and endTime %q must be provided for time range filtering", startTime, endTime) } - startT, err := time.Parse("2006-01-02 15:04:05Z07:00", r.StartTime) + startT, err := time.Parse(timeRangeLayout, startTime) if err != nil { - return fmt.Errorf("parsing startTime: %w", err) + return time.Time{}, time.Time{}, fmt.Errorf("parsing startTime: %w", err) } - endT, err := time.Parse("2006-01-02 15:04:05Z07:00", r.EndTime) + endT, err := time.Parse(timeRangeLayout, endTime) if err != nil { - return fmt.Errorf("parsing endTime: %w", err) + return time.Time{}, time.Time{}, fmt.Errorf("parsing endTime: %w", err) } startTimeUTC := startT.UTC() @@ -57,6 +56,34 @@ func applyRangeFilter(columnName string, r dateRangeFilterParams) error { startTimeUTC, endTimeUTC = endTimeUTC, startTimeUTC } + return startTimeUTC, endTimeUTC, nil +} + +// applyRangeFilter applies a time range filter to the given query based on the provided +// startTime and endTime strings in RFC3339 format. If both are empty and dateRangeDays is greater than zero, +// it filters records updated within the last dateRangeDays days. +func applyRangeFilter(columnName string, r dateRangeFilterParams) error { + + startTimeUTC, endTimeUTC, err := resolveTimeRange(r.DateRangeDays, r.StartTime, r.EndTime) + if err != nil { + return err + } + + if startTimeUTC.IsZero() && endTimeUTC.IsZero() { + return nil + } + + // Without an explicit time range, only the lower boundary is applied so that + // records updated while the query runs are still returned. + if r.StartTime == "" && r.EndTime == "" { + r.Query.Apply( + sm.Where( + psql.Raw(columnName+" > ?", startTimeUTC), + ), + ) + return nil + } + r.Query.Apply( sm.Where( psql.Raw(columnName+" >= ? AND "+columnName+" < ?", startTimeUTC, endTimeUTC), diff --git a/pkg/server/endpoints.go b/pkg/server/endpoints.go index 05340927..32dbbbdf 100644 --- a/pkg/server/endpoints.go +++ b/pkg/server/endpoints.go @@ -193,6 +193,7 @@ func newGinEngine(opts Options) *gin.Engine { r.POST("/api/pipeline/config/targets/search", SearchConfigTargets) r.POST("/api/pipeline/labels/search", SearchLabels) r.POST("/api/pipeline/reports/search", SearchPipelineReports) + r.POST("/api/pipeline/reports/summary", SearchPipelineReportsSummary) r.POST("/api/pipeline/scms/search", SearchSCMs) } else { apiPipeline.POST("/config/sources/search", SearchConfigSources) @@ -200,6 +201,7 @@ func newGinEngine(opts Options) *gin.Engine { apiPipeline.POST("/config/targets/search", SearchConfigTargets) apiPipeline.POST("/labels/search", SearchLabels) apiPipeline.POST("/reports/search", SearchPipelineReports) + apiPipeline.POST("/reports/summary", SearchPipelineReportsSummary) apiPipeline.POST("/scms/search", SearchSCMs) } diff --git a/pkg/server/endpoints_test.go b/pkg/server/endpoints_test.go index 2b6aa0de..72f0b623 100644 --- a/pkg/server/endpoints_test.go +++ b/pkg/server/endpoints_test.go @@ -10,8 +10,10 @@ import ( "net/http" "net/http/httptest" "testing" + "time" "github.com/google/uuid" + "github.com/jackc/pgx/v5" "github.com/stephenafamo/bob/dialect/psql" "github.com/stephenafamo/bob/dialect/psql/dm" "github.com/stretchr/testify/assert" @@ -237,7 +239,7 @@ func TestEndpoints(t *testing.T) { t.Run("GET /api/pipeline/reports/:id", func(t *testing.T) { t.Run("with an unknown report ID", func(t *testing.T) { resp := doGetRequest(t, srv, "/api/pipeline/reports/daa9b61e-42b9-4e35-b9d7-071461a36838") - assert.Equal(t, http.StatusNotFound, resp.StatusCode) + assertErrorResponse(t, resp, http.StatusNotFound, pgx.ErrNoRows.Error()) }) t.Run("with a known report ID", func(t *testing.T) { @@ -484,6 +486,452 @@ func TestEndpoints(t *testing.T) { }, }, removeFieldsAsserter("labels", "created_at", "updated_at", "last_pipeline_report_at")) }) + + // This subtest must run last as it removes every pipeline report. + t.Run("POST /api/pipeline/reports/summary", func(t *testing.T) { + const summaryPath = "/api/pipeline/reports/summary" + // Every bucket identifies itself by its start, formatted as RFC3339, whatever + // the granularity is. + const bucketLayout = time.RFC3339 + + // The previous subtests leave reports behind which would all be + // counted in today's bucket. + truncateReports(t) + t.Cleanup(func() { + truncateReports(t) + }) + + // The expected days are derived from the same clock as the request, so + // a request crossing midnight would make this subtest flaky. Seeding at + // the current time of day keeps that window as small as possible. + now := time.Now().UTC() + + seedReport := func(pipelineResult string, dayOffset int) string { + t.Helper() + + id, err := database.InsertReport(ctx, reports.Report{ + Name: "ci: bump Venom version", + Result: pipelineResult, + ID: "1de1797bbc925e08e473178425b11eb16fc547291f4b45274da24c2b00e2afc3", + PipelineID: "venom", + }) + require.NoError(t, err) + + setReportTimestamp(t, id, now.AddDate(0, 0, dayOffset)) + return id + } + + day := func(dayOffset int) string { + return dayStart(now.AddDate(0, 0, dayOffset)).Format(bucketLayout) + } + + // bucketEntry builds the expected response entry for a bucket, starting from + // a zeroed set of results. + bucketEntry := func(date string, results map[string]any) map[string]any { + allResults := map[string]any{ + "✔": float64(0), + "✗": float64(0), + "⚠": float64(0), + "-": float64(0), + "unknown": float64(0), + } + total := float64(0) + for k, v := range results { + allResults[k] = v + total += v.(float64) + } + + return map[string]any{ + "date": date, + "results": allResults, + "total": total, + } + } + + entry := func(dayOffset int, results map[string]any) map[string]any { + return bucketEntry(day(dayOffset), results) + } + + // want builds the expected response body, the metric and the granularity being + // echoed back by the endpoint. + want := func(granularity string, totalCount float64, entries ...any) map[string]any { + return map[string]any{ + "metric": "result", + "granularity": granularity, + "data": entries, + "total_count": totalCount, + } + } + + seeds := []struct { + result string + offset int + }{ + {"✔", 0}, + {"✔", 0}, + {"✗", 0}, + {"✔", -3}, + // Outside of the default seven days window. + {"⚠", -9}, + } + + seededIDs := make([]string, 0, len(seeds)) + for _, seed := range seeds { + seededIDs = append(seededIDs, seedReport(seed.result, seed.offset)) + } + scmReportID := seededIDs[0] + + // bucketedEntries builds the expected entries of a window of days, bucketing the + // seeded reports the same way the endpoint does. Expressing the expectation this + // way keeps the week and month cases independent from the day this test runs on. + bucketedEntries := func(days int, truncate, next func(time.Time) time.Time) []any { + counts := map[string]map[string]any{} + for _, seed := range seeds { + date := truncate(now.AddDate(0, 0, seed.offset)).Format(bucketLayout) + if counts[date] == nil { + counts[date] = map[string]any{} + } + + previous, _ := counts[date][seed.result].(float64) + counts[date][seed.result] = previous + 1 + } + + entries := []any{} + last := truncate(now) + for bucket := truncate(now.AddDate(0, 0, -(days - 1))); !bucket.After(last); bucket = next(bucket) { + date := bucket.Format(bucketLayout) + entries = append(entries, bucketEntry(date, counts[date])) + } + + return entries + } + + t.Run("with the default window", func(t *testing.T) { + resp := doPostRequest(t, srv, summaryPath, map[string]any{}) + + assertJSONResponse(t, resp, want("day", 4, + entry(-6, nil), + entry(-5, nil), + entry(-4, nil), + entry(-3, map[string]any{"✔": float64(1)}), + entry(-2, nil), + entry(-1, nil), + entry(0, map[string]any{"✔": float64(2), "✗": float64(1)}), + ), assert.Equal) + }) + + t.Run("with an explicit number of days", func(t *testing.T) { + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "days": 3, + }) + + assertJSONResponse(t, resp, want("day", 3, + entry(-2, nil), + entry(-1, nil), + entry(0, map[string]any{"✔": float64(2), "✗": float64(1)}), + ), assert.Equal) + }) + + t.Run("with a window wide enough to catch every report", func(t *testing.T) { + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "days": 10, + }) + + blob := map[string]any{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&blob)) + defer resp.Body.Close() + + assert.Len(t, blob["data"], 10) + assert.Equal(t, float64(5), blob["total_count"]) + assert.Equal(t, entry(-9, map[string]any{"⚠": float64(1)}), blob["data"].([]any)[0]) + }) + + t.Run("with a week granularity", func(t *testing.T) { + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "days": 10, + "granularity": "week", + }) + + assertJSONResponse(t, resp, want("week", 5, + bucketedEntries(10, weekStart, func(t time.Time) time.Time { + return t.AddDate(0, 0, 7) + })..., + ), assert.Equal) + }) + + t.Run("with a month granularity", func(t *testing.T) { + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "days": 10, + "granularity": "month", + }) + + assertJSONResponse(t, resp, want("month", 5, + bucketedEntries(10, monthStart, func(t time.Time) time.Time { + return t.AddDate(0, 1, 0) + })..., + ), assert.Equal) + }) + + t.Run("filtered by scm", func(t *testing.T) { + scmID, err := database.InsertSCM(ctx, "https://example.com/summary.git", "main") + require.NoError(t, err) + t.Cleanup(func() { + deleteSCM(t, scmID) + }) + + attachReportToSCM(t, scmReportID, scmID) + + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "days": 1, + "scmid": scmID, + }) + + assertJSONResponse(t, resp, want("day", 1, + entry(0, map[string]any{"✔": float64(1)}), + ), assert.Equal) + + t.Run("without any scm", func(t *testing.T) { + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "days": 1, + "scmid": "none", + }) + + assertJSONResponse(t, resp, want("day", 2, + entry(0, map[string]any{"✔": float64(1), "✗": float64(1)}), + ), assert.Equal) + }) + }) + + t.Run("with an invalid number of days", func(t *testing.T) { + for _, days := range []int{-1, maxMonitoringDurationDays + 1} { + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "days": days, + }) + + assertErrorResponse(t, resp, http.StatusBadRequest, ErrInvalidDaysParam) + } + }) + + t.Run("with an incomplete time range", func(t *testing.T) { + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "start_time": now.Format(timeRangeLayout), + }) + + assertErrorResponse(t, resp, http.StatusBadRequest, ErrInvalidTimeRangeParams) + }) + + t.Run("with an unsupported metric", func(t *testing.T) { + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "metric": "duration", + }) + + assertErrorResponse(t, resp, http.StatusBadRequest, ErrInvalidMetricParam) + }) + + t.Run("with an unsupported granularity", func(t *testing.T) { + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "granularity": "minute", + }) + + assertErrorResponse(t, resp, http.StatusBadRequest, ErrInvalidGranularityParam) + }) + + t.Run("with both days and hours", func(t *testing.T) { + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "days": 1, + "hours": 1, + }) + + assertErrorResponse(t, resp, http.StatusBadRequest, ErrConflictingWindowParams) + }) + + t.Run("with an invalid number of hours", func(t *testing.T) { + for _, hours := range []int{-1, maxMonitoringDurationDays*24 + 1} { + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "hours": hours, + }) + + assertErrorResponse(t, resp, http.StatusBadRequest, ErrInvalidHoursParam) + } + }) + + t.Run("with a granularity producing too many buckets", func(t *testing.T) { + // The days limit bounds how much of the table is scanned, not how large the + // response gets: a year of hourly buckets is a cheap scan but thousands of + // entries, so it has to be rejected by the bucket limit instead. + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "granularity": "hour", + "days": maxMonitoringDurationDays, + }) + + assertErrorResponse(t, resp, http.StatusBadRequest, ErrTooManyBuckets) + }) + + t.Run("with a time range wider than the limit", func(t *testing.T) { + // The days validation does not cover an explicit time range, so this is + // the only guard against summarizing the whole table. + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "start_time": now.AddDate(0, 0, -(maxMonitoringDurationDays + 1)).Format(timeRangeLayout), + "end_time": now.Format(timeRangeLayout), + }) + + assertErrorResponse(t, resp, http.StatusBadRequest, ErrTimeRangeTooWide) + }) + + // The remaining subtests seed reports of their own, so they must run after the + // ones asserting on the counts above. + t.Run("with a report without any result", func(t *testing.T) { + id := seedReport("", 0) + t.Cleanup(func() { + deleteReport(t, id) + }) + + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "days": 1, + }) + + assertJSONResponse(t, resp, want("day", 4, + entry(0, map[string]any{"✔": float64(2), "✗": float64(1), "unknown": float64(1)}), + ), assert.Equal) + }) + + // This subtest replaces the seeded dataset, so it must run after every other one. + t.Run("with an hour granularity", func(t *testing.T) { + truncateReports(t) + + currentHour := hourStart(now) + + seedReportAt := func(pipelineResult string, at time.Time) { + t.Helper() + + id, err := database.InsertReport(ctx, reports.Report{ + Name: "ci: bump Venom version", + Result: pipelineResult, + ID: "1de1797bbc925e08e473178425b11eb16fc547291f4b45274da24c2b00e2afc3", + PipelineID: "venom", + }) + require.NoError(t, err) + + setReportTimestamp(t, id, at) + } + + // Halfway into each hour, so that a report cannot land in a neighbouring + // bucket. + halfPast := 30 * time.Minute + seedReportAt("✔", currentHour.Add(halfPast)) + seedReportAt("✔", currentHour.Add(-1*time.Hour+halfPast)) + seedReportAt("✗", currentHour.Add(-1*time.Hour+halfPast)) + seedReportAt("⚠", currentHour.Add(-3*time.Hour+halfPast)) + + hour := func(hourOffset int) string { + return currentHour.Add(time.Duration(hourOffset) * time.Hour).Format(bucketLayout) + } + + // Driving this with an explicit time range rather than the hours window keeps + // the expected buckets independent from the clock, so a request crossing an + // hour boundary cannot shift them. + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "granularity": "hour", + "start_time": currentHour.Add(-3 * time.Hour).Format(timeRangeLayout), + "end_time": currentHour.Format(timeRangeLayout), + }) + + assertJSONResponse(t, resp, want("hour", 4, + bucketEntry(hour(-3), map[string]any{"⚠": float64(1)}), + bucketEntry(hour(-2), nil), + bucketEntry(hour(-1), map[string]any{"✔": float64(1), "✗": float64(1)}), + bucketEntry(hour(0), map[string]any{"✔": float64(1)}), + ), assert.Equal) + + t.Run("with a relative hours window", func(t *testing.T) { + // hours is resolved against the server clock, so a request crossing an + // hour boundary shifts the whole window by one bucket. The window is wide + // enough for every seeded report to stay inside it either way, and only + // the shape of the response is asserted. + resp := doPostRequest(t, srv, summaryPath, map[string]any{ + "granularity": "hour", + "hours": 6, + }) + + blob := map[string]any{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&blob)) + defer resp.Body.Close() + + assert.Equal(t, "hour", blob["granularity"]) + assert.Len(t, blob["data"], 6) + assert.Equal(t, float64(4), blob["total_count"]) + }) + }) + }) +} + +// hourStart returns the beginning of the UTC hour of the provided time. +func hourStart(t time.Time) time.Time { + t = t.UTC() + return time.Date(t.Year(), t.Month(), t.Day(), t.Hour(), 0, 0, 0, time.UTC) +} + +// dayStart returns the midnight of the UTC day of the provided time. +func dayStart(t time.Time) time.Time { + t = t.UTC() + return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) +} + +// weekStart returns the monday of the UTC week of the provided time, matching how +// Postgres truncates a timestamp to a week. +func weekStart(t time.Time) time.Time { + t = t.UTC() + day := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) + return day.AddDate(0, 0, -((int(day.Weekday()) + 6) % 7)) +} + +// monthStart returns the first day of the UTC month of the provided time. +func monthStart(t time.Time) time.Time { + t = t.UTC() + return time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, time.UTC) +} + +// timeRangeLayout is the layout the start_time and end_time filters are expected in. +const timeRangeLayout = "2006-01-02 15:04:05Z07:00" + +// truncateReports removes every pipeline report from the database. +func truncateReports(t *testing.T) { + t.Helper() + + _, err := database.DB.Exec(context.TODO(), "DELETE FROM pipelineReports") + require.NoError(t, err) +} + +// deleteReport removes a single pipeline report from the database. +func deleteReport(t *testing.T, id string) { + t.Helper() + + _, err := database.DB.Exec(context.TODO(), "DELETE FROM pipelineReports WHERE id = $1", id) + require.NoError(t, err) +} + +// setReportTimestamp forces the creation and update date of an existing report. +// InsertReport always relies on the database defaults, so backdating a report +// requires updating it afterwards. +func setReportTimestamp(t *testing.T, id string, at time.Time) { + t.Helper() + + // The value must be normalized to UTC: the driver sends the wall clock of + // its own location, which is what the timestamp column stores. + _, err := database.DB.Exec(context.TODO(), + "UPDATE pipelineReports SET created_at = $1, updated_at = $1 WHERE id = $2", + at.UTC(), id) + require.NoError(t, err) +} + +// attachReportToSCM associates an existing report to an scm. +func attachReportToSCM(t *testing.T, reportID, scmID string) { + t.Helper() + + _, err := database.DB.Exec(context.TODO(), + "UPDATE pipelineReports SET target_db_scm_ids = ARRAY[$1]::uuid[] WHERE id = $2", + scmID, reportID) + require.NoError(t, err) } func doGetRequest(t *testing.T, ts *httptest.Server, path string) *http.Response { diff --git a/pkg/server/labeldb_handlers.go b/pkg/server/labeldb_handlers.go index fc41ef4c..79328239 100644 --- a/pkg/server/labeldb_handlers.go +++ b/pkg/server/labeldb_handlers.go @@ -112,10 +112,36 @@ func ListLabels(c *gin.Context) { } } +// SearchLabelsRequest represents the filters used to search labels. +type SearchLabelsRequest struct { + // Id is the unique identifier of the label. + Id string `json:"id"` + // Key is the key of the label. + Key string `json:"key"` + // Value is the value of the label. + Value string `json:"value"` + // Limit is the maximum number of labels to return + // This is optional and can be used to limit the number of labels returned + Limit int `json:"limit"` + // Page is the page number for pagination + // This is optional and can be used to paginate the results + Page int `json:"page"` + // StartTime is the start time for the time range filter + // This is optional and can be used to filter labels by a specific start time + // Time format is RFC3339: 2006-01-02T15:04:05Z07:00 + StartTime string `json:"start_time"` + // EndTime is the end time for the time range filter + // This is optional and can be used to filter labels by a specific end time + // Time format is RFC3339: 2006-01-02T15:04:05Z07:00 + EndTime string `json:"end_time"` + // KeyOnly specifies if we only need to retrieve a list of uniq Label keys + KeyOnly bool `json:"key_only"` +} + // SearchLabels searches labels from the database using advanced filtering // @Summary Search labels // @Description Search labels in the database using advanced filtering -// @Param body body queryData true "Search parameters" +// @Param body body SearchLabelsRequest true "Search parameters" // @Tags Labels // @Accept json // @Produce json @@ -125,32 +151,7 @@ func ListLabels(c *gin.Context) { // @Router /api/pipeline/labels/search [post] func SearchLabels(c *gin.Context) { - type queryData struct { - // Id is the unique identifier of the label. - Id string `json:"id"` - // Key is the key of the label. - Key string `json:"key"` - // Value is the value of the label. - Value string `json:"value"` - // Limit is the maximum number of labels to return - // This is optional and can be used to limit the number of labels returned - Limit int `json:"limit"` - // Page is the page number for pagination - // This is optional and can be used to paginate the results - Page int `json:"page"` - // StartTime is the start time for the time range filter - // This is optional and can be used to filter labels by a specific start time - // Time format is RFC3339: 2006-01-02T15:04:05Z07:00 - StartTime string `json:"start_time"` - // EndTime is the end time for the time range filter - // This is optional and can be used to filter labels by a specific end time - // Time format is RFC3339: 2006-01-02T15:04:05Z07:00 - EndTime string `json:"end_time"` - // KeyOnly specifies if we only need to retrieve a list of uniq Label keys - KeyOnly bool `json:"key_only"` - } - - queryParams := queryData{} + queryParams := SearchLabelsRequest{} if err := c.ShouldBindJSON(&queryParams); err != nil { logrus.Errorf("failed to read json body: %s", err) diff --git a/pkg/server/report_handlers.go b/pkg/server/report_handlers.go index ae0e507e..765db2f9 100644 --- a/pkg/server/report_handlers.go +++ b/pkg/server/report_handlers.go @@ -173,6 +173,184 @@ func SearchPipelineReports(c *gin.Context) { }) } +// SearchPipelineReportsSummaryRequest represents the filters used to summarize +// pipeline reports. +type SearchPipelineReportsSummaryRequest struct { + // Metric is what the reports are counted by. It defaults to "result", which is + // the only value supported so far. + Metric string `json:"metric,omitempty"` + // Granularity is the size of the time buckets, one of "hour", "day", "week" or + // "month". It defaults to "day". + Granularity string `json:"granularity,omitempty"` + // Days is the number of days to summarize, today included. + // It defaults to 7 and is ignored when hours, or start_time and end_time, are provided. + Days int `json:"days,omitempty"` + // Hours is the number of hours to summarize, the current hour included. + // It cannot be combined with days and is ignored when start_time and end_time are provided. + Hours int `json:"hours,omitempty"` + // ScmID is the ID of the SCM to filter reports by. + // Use "none" to only count the reports which are not attached to any SCM. + ScmID string `json:"scmid,omitempty"` + // Labels is a map of labels to filter reports by. + Labels map[string]string `json:"labels,omitempty"` + // StartTime is the start time for the time range filter. + // Time format is: 2006-01-02 15:04:05Z07:00 + StartTime string `json:"start_time,omitempty"` + // EndTime is the end time for the time range filter. + // Time format is: 2006-01-02 15:04:05Z07:00 + EndTime string `json:"end_time,omitempty"` +} + +// SearchPipelineReportsSummaryResponse represents the response for the +// SearchPipelineReportsSummary endpoint. +type SearchPipelineReportsSummaryResponse struct { + // Metric is the metric the reports were counted by. + Metric string `json:"metric"` + // Granularity is the size of the time buckets of the entries. + Granularity string `json:"granularity"` + // Data contains one entry per time bucket, ordered from the oldest to the most recent one. + Data []database.ReportResultSummaryEntry `json:"data"` + // TotalCount is the total number of reports matching the query. + TotalCount int `json:"total_count"` +} + +// SearchPipelineReportsSummary returns the number of pipeline reports per result, per time bucket. +// @Summary Summarize pipeline reports +// @Description Return the number of pipeline reports per result for each time bucket of the requested time range. +// @Description Buckets are UTC hours, UTC calendar days, ISO weeks or calendar months depending on the +// @Description granularity, and the date of an entry is the start of its bucket, formatted as RFC3339. +// @Description Every report is counted, including several reports of the same pipeline, and buckets without +// @Description any report are returned with a zeroed entry. +// @Tags Pipeline Reports +// @Accept json +// @Produce json +// @Param body body SearchPipelineReportsSummaryRequest true "Summary filters" +// @Success 200 {object} SearchPipelineReportsSummaryResponse +// @Failure 400 {object} DefaultResponseModel +// @Failure 500 {object} DefaultResponseModel +// @Router /api/pipeline/reports/summary [post] +func SearchPipelineReportsSummary(c *gin.Context) { + queryParams := SearchPipelineReportsSummaryRequest{} + + if err := c.ShouldBindJSON(&queryParams); err != nil { + logrus.Errorf("failed to read json body: %s", err) + c.JSON(http.StatusBadRequest, DefaultResponseModel{ + Err: err.Error(), + }) + return + } + + metric := queryParams.Metric + if metric == "" { + metric = summaryMetricResult + } + + if metric != summaryMetricResult { + c.JSON(http.StatusBadRequest, DefaultResponseModel{ + Err: ErrInvalidMetricParam, + }) + return + } + + granularity := database.SummaryGranularity(queryParams.Granularity) + if granularity == "" { + granularity = database.SummaryGranularityDay + } + + if !granularity.IsValid() { + c.JSON(http.StatusBadRequest, DefaultResponseModel{ + Err: ErrInvalidGranularityParam, + }) + return + } + + if queryParams.Days != 0 && queryParams.Hours != 0 { + c.JSON(http.StatusBadRequest, DefaultResponseModel{ + Err: ErrConflictingWindowParams, + }) + return + } + + hours := queryParams.Hours + if hours < 0 || hours > maxMonitoringDurationDays*24 { + c.JSON(http.StatusBadRequest, DefaultResponseModel{ + Err: ErrInvalidHoursParam, + }) + return + } + + days := queryParams.Days + switch { + case days == 0: + // Only fall back to the default window when no window was asked for at all, + // otherwise it would silently override hours. + if hours == 0 { + days = monitoringDurationDays + } + case days < 0 || days > maxMonitoringDurationDays: + c.JSON(http.StatusBadRequest, DefaultResponseModel{ + Err: ErrInvalidDaysParam, + }) + return + } + + // Catching this here returns a 400 instead of the 500 that the database layer + // would return for the same mistake. + if (queryParams.StartTime == "") != (queryParams.EndTime == "") { + c.JSON(http.StatusBadRequest, DefaultResponseModel{ + Err: ErrInvalidTimeRangeParams, + }) + return + } + + dataset, totalCount, err := database.SearchReportsSummary( + database.ReportSummaryParams{ + Ctx: c, + Days: days, + Hours: hours, + Granularity: granularity, + MaxDays: maxMonitoringDurationDays, + MaxBuckets: maxSummaryBuckets, + ScmID: queryParams.ScmID, + Labels: queryParams.Labels, + StartTime: queryParams.StartTime, + EndTime: queryParams.EndTime, + }, + ) + if err != nil { + // An explicit time range bypasses the days validation above, so this is the + // only place a range wider than the limit can be caught. + if errors.Is(err, database.ErrSummaryRangeTooWide) { + c.JSON(http.StatusBadRequest, DefaultResponseModel{ + Err: ErrTimeRangeTooWide, + }) + return + } + + // The number of buckets depends on the granularity, which the validation above + // cannot account for on its own. + if errors.Is(err, database.ErrSummaryTooManyBuckets) { + c.JSON(http.StatusBadRequest, DefaultResponseModel{ + Err: ErrTooManyBuckets, + }) + return + } + + logrus.Errorf("summarizing reports: %s", err) + c.JSON(http.StatusInternalServerError, DefaultResponseModel{ + Err: err.Error(), + }) + return + } + + c.JSON(http.StatusOK, SearchPipelineReportsSummaryResponse{ + Metric: metric, + Granularity: string(granularity), + Data: dataset, + TotalCount: totalCount, + }) +} + // ListPipelineReports returns all pipeline reports from the database // @Summary List all pipeline reports // @Description List all pipeline reports from the database diff --git a/pkg/server/var.go b/pkg/server/var.go index cd2159ce..8e0cded9 100644 --- a/pkg/server/var.go +++ b/pkg/server/var.go @@ -6,6 +6,15 @@ var ( // performance of the database queries. // The goal is to minimize the impact in small environment monitoringDurationDays int = 7 + // maxMonitoringDurationDays is the largest number of days a summary query may span. + // The time range itself is indexed but the aggregation runs over every matching row, + // so a wide window means scanning most of the table. + maxMonitoringDurationDays int = 366 + // maxSummaryBuckets is the largest number of buckets a summary may return. + // maxMonitoringDurationDays bounds how much of the table a summary scans, this bounds + // how large its response gets: an hourly summary of a year is a cheap scan but would + // return more than eight thousand entries. + maxSummaryBuckets int = 1000 // errMessageType is the key used in JSON responses to indicate an error message. errMessageType = "error" // successMessageType is used to indicate a successful operation in API responses. @@ -19,5 +28,27 @@ const ( ErrInvalidSummaryParam = "invalid summary parameter" // ErrInvalidKeyOnlyParam is the error message returned when the keyonly parameter is invalid. ErrInvalidKeyOnlyParam = "invalid keyonly parameter" - ErrInvalidJWT = "JWT is invalid" + // ErrInvalidDaysParam is the error message returned when the days parameter is out of range. + ErrInvalidDaysParam = "invalid days parameter" + // ErrInvalidTimeRangeParams is the error message returned when only one of the time range boundaries is provided. + ErrInvalidTimeRangeParams = "both start_time and end_time must be provided" + // ErrInvalidMetricParam is the error message returned when the requested summary metric is not supported. + ErrInvalidMetricParam = "invalid metric parameter" + // ErrInvalidGranularityParam is the error message returned when the requested summary granularity is not supported. + ErrInvalidGranularityParam = "invalid granularity parameter" + // ErrTimeRangeTooWide is the error message returned when the requested time range spans more + // than maxMonitoringDurationDays days. + ErrTimeRangeTooWide = "requested time range exceeds the maximum allowed span" + // ErrInvalidHoursParam is the error message returned when the hours parameter is out of range. + ErrInvalidHoursParam = "invalid hours parameter" + // ErrConflictingWindowParams is the error message returned when both days and hours are provided. + ErrConflictingWindowParams = "days and hours cannot be combined" + // ErrTooManyBuckets is the error message returned when the requested time range and granularity + // would produce more than maxSummaryBuckets entries. + ErrTooManyBuckets = "requested time range and granularity produce too many buckets" + ErrInvalidJWT = "JWT is invalid" + + // summaryMetricResult counts the pipeline reports per Updatecli result. It is the + // only metric supported by the reports summary so far. + summaryMetricResult = "result" ) From 9097f468b5646c1a893f7ca4ced55efacb5edc1d Mon Sep 17 00:00:00 2001 From: Olivier Vernin Date: Wed, 5 Aug 2026 20:46:47 +0200 Subject: [PATCH 2/3] feat: allow to filter based on report result Signed-off-by: Olivier Vernin --- pkg/database/database_test.go | 2 +- pkg/database/report.go | 33 ++++++++++++++++++++++++++++++++- pkg/database/scm.go | 23 ++++++++++++++++++----- pkg/server/endpoints_test.go | 2 +- pkg/server/report_handlers.go | 10 ++++++++++ pkg/server/scmdb_handlers.go | 9 +++++++-- 6 files changed, 69 insertions(+), 10 deletions(-) diff --git a/pkg/database/database_test.go b/pkg/database/database_test.go index 3fc814fa..d31b2af7 100644 --- a/pkg/database/database_test.go +++ b/pkg/database/database_test.go @@ -66,7 +66,7 @@ func TestDatabase(t *testing.T) { }) t.Run("migration 000010 backfills pipeline_result", func(t *testing.T) { - // Migration 000004 read "data ->> 'result'" while a marshalled report stores + // Migration 000004 read "data ->> 'result'" while a marshaled report stores // the key as "Result", so its backfill silently did nothing and every report // inserted before it still has an empty pipeline_result. id, err := InsertReport(ctx, reports.Report{ diff --git a/pkg/database/report.go b/pkg/database/report.go index bbe44528..396d670f 100644 --- a/pkg/database/report.go +++ b/pkg/database/report.go @@ -99,6 +99,9 @@ type SearchLatestReportsParams struct { Page int Latest bool Labels map[string]string + // Results restricts the search to the reports whose pipeline result is one of + // them. An empty list does not filter anything out. + Results []string } // SearchLatestReports searches the latest reports according some parameters. @@ -172,6 +175,8 @@ func SearchLatestReports(params SearchLatestReportsParams) ([]SearchLatestReport return nil, 0, err } + applyResultFilter(&query, params.Results) + // Total counter query must be built before applying pagination // because it needs to count all the reports matching the query. totalCountQuery := psql.Select(sm.From(query), sm.Columns("count(*)")) @@ -364,6 +369,9 @@ type ReportSummaryParams struct { ScmID string // Labels restricts the summary to the reports matching those labels. Labels map[string]string + // Results restricts the summary to the reports whose pipeline result is one of + // them. An empty list does not filter anything out. + Results []string } // ReportResultSummaryEntry contains the number of reports per result for a single time bucket. @@ -423,6 +431,8 @@ func SearchReportsSummary(params ReportSummaryParams) ([]ReportResultSummaryEntr return nil, 0, err } + applyResultFilter(&query, params.Results) + if len(params.Labels) > 0 { // The report window is widened to whole buckets so the label lookup must cover // the same range, otherwise labels timestamped within the widened part would be @@ -520,7 +530,8 @@ func summaryResultKey(pipelineResult string) string { // summary. Both are the start of a bucket, in UTC. func summaryRange(params ReportSummaryParams, granularity SummaryGranularity) (time.Time, time.Time, error) { - firstTime, lastTime := time.Time{}, time.Time{} + var firstTime time.Time + var lastTime time.Time switch { case params.StartTime != "" || params.EndTime != "": @@ -966,6 +977,26 @@ func applyResourceConfigFilter(query *bob.BaseQuery[*dialect.SelectQuery], id, k return nil } +// applyResultFilter restricts the given query to the reports whose pipeline result is +// one of those given. An empty list does not filter anything out. +// +// pipeline_result is denormalized from data ->> 'Result' when the report is inserted, +// and indexed alongside updated_at, so this does not have to reach into the jsonb +// payload. A result which is not an Updatecli one simply matches no report, rather +// than being silently dropped from the filter. +func applyResultFilter(query *bob.BaseQuery[*dialect.SelectQuery], results []string) { + if len(results) == 0 { + return + } + + args := make([]bob.Expression, len(results)) + for i := range results { + args[i] = psql.Arg(results[i]) + } + + query.Apply(sm.Where(psql.Quote("pipeline_result").In(args...))) +} + // applyScmFilter restricts the given query to the reports associated to a specific scm. // An empty scmID does not filter anything while "none", "null", or "nil" only keeps // the reports which are not associated to any scm. diff --git a/pkg/database/scm.go b/pkg/database/scm.go index d5be685b..37f85b97 100644 --- a/pkg/database/scm.go +++ b/pkg/database/scm.go @@ -3,6 +3,7 @@ package database import ( "context" "fmt" + "slices" "github.com/google/uuid" "github.com/sirupsen/logrus" @@ -14,7 +15,6 @@ import ( ) // InsertSCM creates a new SCM and inserts it into the database. -// // It returns the ID of the newly created SCM. func InsertSCM(ctx context.Context, url, branch string) (string, error) { //"INSERT INTO scms (url, branch) VALUES ($1, $2) RETURNING id" @@ -151,10 +151,13 @@ type GetSCMSummaryParams struct { StartTime string EndTime string Labels map[string]string - TotalCount int - TotalActions int - Ctx context.Context - ScmRows []model.SCM + // Results restricts the summary to the reports whose pipeline result is one of + // them. An empty list does not filter anything out. + Results []string + TotalCount int + TotalActions int + Ctx context.Context + ScmRows []model.SCM } // GetSCMSummary returns a list of scms summary from the scm database table. @@ -255,6 +258,16 @@ func GetSCMSummary(params GetSCMSummaryParams) (*SCMDataset, error) { return nil, fmt.Errorf("scanning scm summary row: %w", err) } + // The results are dropped here rather than in the query above on purpose. + // That query keeps the latest report of every pipeline, so this summary + // reports where each pipeline stands now; filtering the reports before + // that would instead keep the latest report which happened to carry one + // of those results, reporting a pipeline as failing long after it + // recovered. + if len(params.Results) > 0 && !slices.Contains(params.Results, result) { + continue + } + resultFound := false for r := range dataset.Data[scmURL][scmBranch].TotalResultByType { if r == result { diff --git a/pkg/server/endpoints_test.go b/pkg/server/endpoints_test.go index 72f0b623..b55a42ab 100644 --- a/pkg/server/endpoints_test.go +++ b/pkg/server/endpoints_test.go @@ -815,7 +815,7 @@ func TestEndpoints(t *testing.T) { setReportTimestamp(t, id, at) } - // Halfway into each hour, so that a report cannot land in a neighbouring + // Halfway into each hour, so that a report cannot land in a neighboring // bucket. halfPast := 30 * time.Minute seedReportAt("✔", currentHour.Add(halfPast)) diff --git a/pkg/server/report_handlers.go b/pkg/server/report_handlers.go index 765db2f9..18355b81 100644 --- a/pkg/server/report_handlers.go +++ b/pkg/server/report_handlers.go @@ -131,6 +131,10 @@ func SearchPipelineReports(c *gin.Context) { Latest bool `json:"latest"` // Labels is a map of labels to filter reports by Labels map[string]string `json:"labels,omitempty"` + // Results is a list of pipeline results to filter reports by, such as + // "✔", "✗", "⚠" or "-". A report matches when its result is any of them. + // This is optional and an empty list does not filter anything out. + Results []string `json:"results,omitempty"` } queryParams := queryData{} @@ -157,6 +161,7 @@ func SearchPipelineReports(c *gin.Context) { Page: queryParams.Page, Latest: queryParams.Latest, Labels: queryParams.Labels, + Results: queryParams.Results, }, ) if err != nil { @@ -193,6 +198,10 @@ type SearchPipelineReportsSummaryRequest struct { ScmID string `json:"scmid,omitempty"` // Labels is a map of labels to filter reports by. Labels map[string]string `json:"labels,omitempty"` + // Results is a list of pipeline results to filter reports by, such as + // "✔", "✗", "⚠" or "-". A report is counted when its result is any of them. + // An empty list does not filter anything out. + Results []string `json:"results,omitempty"` // StartTime is the start time for the time range filter. // Time format is: 2006-01-02 15:04:05Z07:00 StartTime string `json:"start_time,omitempty"` @@ -313,6 +322,7 @@ func SearchPipelineReportsSummary(c *gin.Context) { MaxBuckets: maxSummaryBuckets, ScmID: queryParams.ScmID, Labels: queryParams.Labels, + Results: queryParams.Results, StartTime: queryParams.StartTime, EndTime: queryParams.EndTime, }, diff --git a/pkg/server/scmdb_handlers.go b/pkg/server/scmdb_handlers.go index 991dc708..9dedf57a 100644 --- a/pkg/server/scmdb_handlers.go +++ b/pkg/server/scmdb_handlers.go @@ -30,6 +30,9 @@ type SearchSCMsRequest struct { EndTime string `json:"end_time"` // Labels filters SCM summaries by report labels. Labels map[string]string `json:"labels,omitempty"` + // Results filters SCM summaries by pipeline result, such as "✔", "✗", "⚠" or + // "-". An empty list does not filter anything out. + Results []string `json:"results,omitempty"` // URL is the SCM URL to filter by. URL string `json:"url,omitempty"` // Branch is the SCM branch to filter by. @@ -82,6 +85,7 @@ func SearchSCMs(c *gin.Context) { queryParams.StartTime, queryParams.EndTime, queryParams.Labels, + queryParams.Results, ) return } @@ -154,7 +158,7 @@ func ListSCMs(c *gin.Context) { } if summary { - findSCMSummary(c, rows, totalCount, queryValues.Get("start_time"), queryValues.Get("end_time"), map[string]string{}) + findSCMSummary(c, rows, totalCount, queryValues.Get("start_time"), queryValues.Get("end_time"), map[string]string{}, nil) return } @@ -182,7 +186,7 @@ type FindSCMSummaryResponse struct { } // findSCMSummary returns a summary of all git repositories detected. -func findSCMSummary(c *gin.Context, scmRows []model.SCM, totalCount int, startTime, endTime string, labels map[string]string) { +func findSCMSummary(c *gin.Context, scmRows []model.SCM, totalCount int, startTime, endTime string, labels map[string]string, results []string) { var data map[string]database.SCMBranchDataset dataset, err := database.GetSCMSummary(database.GetSCMSummaryParams{ @@ -193,6 +197,7 @@ func findSCMSummary(c *gin.Context, scmRows []model.SCM, totalCount int, startTi StartTime: startTime, EndTime: endTime, Labels: labels, + Results: results, }) if err != nil { logrus.Errorf("getting scm summary failed: %s", err) From b8abcd018dc4306205c3d6a4537546aebfbd76d5 Mon Sep 17 00:00:00 2001 From: Olivier Vernin Date: Thu, 6 Aug 2026 10:12:56 +0200 Subject: [PATCH 3/3] feat: allow to filter by open/close action Signed-off-by: Olivier Vernin --- docs/docs.go | 29 +++ docs/swagger.json | 29 +++ docs/swagger.yaml | 45 ++++ pkg/database/database_test.go | 102 ++++++++ ...alter_pipelineReports_open_action.down.sql | 5 + ...1_alter_pipelineReports_open_action.up.sql | 29 +++ pkg/database/report.go | 77 +++++- pkg/database/scm.go | 41 ++- pkg/server/endpoints_test.go | 246 +++++++++++++++++- pkg/server/report_handlers.go | 19 ++ pkg/server/scmdb_handlers.go | 67 +++-- 11 files changed, 650 insertions(+), 39 deletions(-) create mode 100644 pkg/database/migrations/000011_alter_pipelineReports_open_action.down.sql create mode 100644 pkg/database/migrations/000011_alter_pipelineReports_open_action.up.sql diff --git a/docs/docs.go b/docs/docs.go index 669b9c37..5493305f 100644 --- a/docs/docs.go +++ b/docs/docs.go @@ -1048,6 +1048,13 @@ const docTemplate = `{ "description": "Date is the start of the bucket, in UTC, formatted as RFC3339.", "type": "string" }, + "open_actions": { + "description": "OpenActions contains, for each Updatecli result, how many of the reports counted in\nResults also carry an open action, such as a pull request still waiting to be merged.\nIt is a breakdown of Results, not an addition to it, so its counts are always lower\nthan or equal to the matching ones in Results.\n\nThe interesting one is the count reported under the success result: those pipelines\nran fine and had nothing to change only because the change is already waiting in a\npull request.", + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, "results": { "description": "Results contains the number of reports per Updatecli result for that bucket.", "type": "object", @@ -1907,6 +1914,17 @@ const docTemplate = `{ "description": "Metric is what the reports are counted by. It defaults to \"result\", which is\nthe only value supported so far.", "type": "string" }, + "open_action": { + "description": "OpenAction filters reports by whether they carry an action left open, such as a\npull request still waiting to be merged. This is optional: unset does not filter\nanything out, true only counts the reports with an open action and false only the\nones without.\n\nThe same breakdown is reported without filtering anything out under the open_actions\nkey of every bucket.", + "type": "boolean" + }, + "results": { + "description": "Results is a list of pipeline results to filter reports by, such as\n\"✔\", \"✗\", \"⚠\" or \"-\". A report is counted when its result is any of them.\nAn empty list does not filter anything out.", + "type": "array", + "items": { + "type": "string" + } + }, "scmid": { "description": "ScmID is the ID of the SCM to filter reports by.\nUse \"none\" to only count the reports which are not attached to any SCM.", "type": "string" @@ -1963,10 +1981,21 @@ const docTemplate = `{ "description": "Limit is the maximum number of SCMs to return.", "type": "integer" }, + "open_action": { + "description": "OpenAction filters SCM summaries by whether a pipeline carries an action left open,\nsuch as a pull request still waiting to be merged. This is optional: unset does not\nfilter anything out, true only keeps the pipelines with an open action and false only\nthe ones without.", + "type": "boolean" + }, "page": { "description": "Page is the page number for pagination.", "type": "integer" }, + "results": { + "description": "Results filters SCM summaries by pipeline result, such as \"✔\", \"✗\", \"⚠\" or\n\"-\". An empty list does not filter anything out.", + "type": "array", + "items": { + "type": "string" + } + }, "scmid": { "description": "ScmID is the ID of the SCM to filter by.", "type": "string" diff --git a/docs/swagger.json b/docs/swagger.json index 97557645..81d6d0b3 100644 --- a/docs/swagger.json +++ b/docs/swagger.json @@ -1037,6 +1037,13 @@ "description": "Date is the start of the bucket, in UTC, formatted as RFC3339.", "type": "string" }, + "open_actions": { + "description": "OpenActions contains, for each Updatecli result, how many of the reports counted in\nResults also carry an open action, such as a pull request still waiting to be merged.\nIt is a breakdown of Results, not an addition to it, so its counts are always lower\nthan or equal to the matching ones in Results.\n\nThe interesting one is the count reported under the success result: those pipelines\nran fine and had nothing to change only because the change is already waiting in a\npull request.", + "type": "object", + "additionalProperties": { + "type": "integer" + } + }, "results": { "description": "Results contains the number of reports per Updatecli result for that bucket.", "type": "object", @@ -1896,6 +1903,17 @@ "description": "Metric is what the reports are counted by. It defaults to \"result\", which is\nthe only value supported so far.", "type": "string" }, + "open_action": { + "description": "OpenAction filters reports by whether they carry an action left open, such as a\npull request still waiting to be merged. This is optional: unset does not filter\nanything out, true only counts the reports with an open action and false only the\nones without.\n\nThe same breakdown is reported without filtering anything out under the open_actions\nkey of every bucket.", + "type": "boolean" + }, + "results": { + "description": "Results is a list of pipeline results to filter reports by, such as\n\"✔\", \"✗\", \"⚠\" or \"-\". A report is counted when its result is any of them.\nAn empty list does not filter anything out.", + "type": "array", + "items": { + "type": "string" + } + }, "scmid": { "description": "ScmID is the ID of the SCM to filter reports by.\nUse \"none\" to only count the reports which are not attached to any SCM.", "type": "string" @@ -1952,10 +1970,21 @@ "description": "Limit is the maximum number of SCMs to return.", "type": "integer" }, + "open_action": { + "description": "OpenAction filters SCM summaries by whether a pipeline carries an action left open,\nsuch as a pull request still waiting to be merged. This is optional: unset does not\nfilter anything out, true only keeps the pipelines with an open action and false only\nthe ones without.", + "type": "boolean" + }, "page": { "description": "Page is the page number for pagination.", "type": "integer" }, + "results": { + "description": "Results filters SCM summaries by pipeline result, such as \"✔\", \"✗\", \"⚠\" or\n\"-\". An empty list does not filter anything out.", + "type": "array", + "items": { + "type": "string" + } + }, "scmid": { "description": "ScmID is the ID of the SCM to filter by.", "type": "string" diff --git a/docs/swagger.yaml b/docs/swagger.yaml index 4cba9546..289e03a8 100644 --- a/docs/swagger.yaml +++ b/docs/swagger.yaml @@ -75,6 +75,19 @@ definitions: date: description: Date is the start of the bucket, in UTC, formatted as RFC3339. type: string + open_actions: + additionalProperties: + type: integer + description: |- + OpenActions contains, for each Updatecli result, how many of the reports counted in + Results also carry an open action, such as a pull request still waiting to be merged. + It is a breakdown of Results, not an addition to it, so its counts are always lower + than or equal to the matching ones in Results. + + The interesting one is the count reported under the success result: those pipelines + ran fine and had nothing to change only because the change is already waiting in a + pull request. + type: object results: additionalProperties: type: integer @@ -702,6 +715,24 @@ definitions: Metric is what the reports are counted by. It defaults to "result", which is the only value supported so far. type: string + open_action: + description: |- + OpenAction filters reports by whether they carry an action left open, such as a + pull request still waiting to be merged. This is optional: unset does not filter + anything out, true only counts the reports with an open action and false only the + ones without. + + The same breakdown is reported without filtering anything out under the open_actions + key of every bucket. + type: boolean + results: + description: |- + Results is a list of pipeline results to filter reports by, such as + "✔", "✗", "⚠" or "-". A report is counted when its result is any of them. + An empty list does not filter anything out. + items: + type: string + type: array scmid: description: |- ScmID is the ID of the SCM to filter reports by. @@ -749,9 +780,23 @@ definitions: limit: description: Limit is the maximum number of SCMs to return. type: integer + open_action: + description: |- + OpenAction filters SCM summaries by whether a pipeline carries an action left open, + such as a pull request still waiting to be merged. This is optional: unset does not + filter anything out, true only keeps the pipelines with an open action and false only + the ones without. + type: boolean page: description: Page is the page number for pagination. type: integer + results: + description: |- + Results filters SCM summaries by pipeline result, such as "✔", "✗", "⚠" or + "-". An empty list does not filter anything out. + items: + type: string + type: array scmid: description: ScmID is the ID of the SCM to filter by. type: string diff --git a/pkg/database/database_test.go b/pkg/database/database_test.go index d31b2af7..e6d84885 100644 --- a/pkg/database/database_test.go +++ b/pkg/database/database_test.go @@ -101,4 +101,106 @@ func TestDatabase(t *testing.T) { assert.Equal(t, result.SUCCESS, pipelineResult) assert.Equal(t, "ci: bump Venom version", pipelineName) }) + + t.Run("openActionSQLExpr detects an action left open", func(t *testing.T) { + // This is the contract the whole open action dimension rests on: Updatecli reports + // a pipeline which had nothing to change as a success even when its change is + // already waiting in an open pull request, and the only trace of it in the payload + // is reports.Action.Link, serialized as "actionUrl" and omitted when empty. + // + // The expression is exercised through the reports it is meant to tell apart rather + // than through a handcrafted jsonb document, so that a change to the Action struct + // of the Updatecli module this repository depends on breaks this test. + testdata := []struct { + name string + report reports.Report + want bool + }{ + { + name: "success with a pull request left open", + report: reports.Report{ + Name: "succeeded, pull request still open", + Result: result.SUCCESS, + ID: "open-action-success", + Actions: map[string]*reports.Action{ + "default": { + ID: "default", + Link: "https://github.com/updatecli/udash/pull/42", + }, + }, + }, + want: true, + }, + { + name: "success with an action but no pull request", + report: reports.Report{ + Name: "succeeded, nothing to follow up", + Result: result.SUCCESS, + ID: "no-open-action-success", + Actions: map[string]*reports.Action{ + "default": {ID: "default"}, + }, + }, + want: false, + }, + { + name: "pipeline without any action configured", + report: reports.Report{ + Name: "no action configured", + Result: result.SUCCESS, + ID: "no-action-at-all", + }, + want: false, + }, + { + name: "attention with a pull request left open", + report: reports.Report{ + Name: "changed something and opened a pull request", + Result: result.ATTENTION, + ID: "open-action-attention", + Actions: map[string]*reports.Action{ + "default": { + ID: "default", + Link: "https://github.com/updatecli/udash/pull/43", + }, + }, + }, + want: true, + }, + } + + for _, tt := range testdata { + t.Run(tt.name, func(t *testing.T) { + id, err := InsertReport(ctx, tt.report) + require.NoError(t, err) + t.Cleanup(func() { + _, err := DB.Exec(ctx, "DELETE FROM pipelineReports WHERE id = $1", id) + assert.NoError(t, err) + }) + + got := false + require.NoError(t, DB.QueryRow(ctx, + "SELECT "+openActionSQLExpr+" FROM pipelineReports WHERE id = $1", id, + ).Scan(&got)) + + assert.Equal(t, tt.want, got) + }) + } + }) + + t.Run("migration 000011 indexes the open action expression", func(t *testing.T) { + // The jsonpath is inlined in openActionSQLExpr so that it matches the index + // expression. Binding it as a parameter would still return the right reports while + // silently falling back to a sequential scan over every stored payload. + indexed := false + require.NoError(t, DB.QueryRow(ctx, ` + SELECT count(*) = 1 + FROM pg_indexes + WHERE tablename = 'pipelinereports' + AND indexname = 'idx_pipelinereports_updated_at_result_open_action' + AND indexdef LIKE '%jsonb_path_exists%'`, + ).Scan(&indexed)) + + assert.True(t, indexed) + }) } diff --git a/pkg/database/migrations/000011_alter_pipelineReports_open_action.down.sql b/pkg/database/migrations/000011_alter_pipelineReports_open_action.down.sql new file mode 100644 index 00000000..978279bf --- /dev/null +++ b/pkg/database/migrations/000011_alter_pipelineReports_open_action.down.sql @@ -0,0 +1,5 @@ +BEGIN; + +DROP INDEX IF EXISTS idx_pipelinereports_updated_at_result_open_action; + +COMMIT; diff --git a/pkg/database/migrations/000011_alter_pipelineReports_open_action.up.sql b/pkg/database/migrations/000011_alter_pipelineReports_open_action.up.sql new file mode 100644 index 00000000..901a3036 --- /dev/null +++ b/pkg/database/migrations/000011_alter_pipelineReports_open_action.up.sql @@ -0,0 +1,29 @@ +-- Updatecli reports a pipeline which had nothing to change as a success, even when the +-- change it would have made is already sitting in a pull request nobody merged. That state +-- is the one which needs a human, yet it is indistinguishable from a genuinely up to date +-- pipeline when looking at the result alone. +-- +-- It is however recorded in the report payload: reports.Action.Link is serialized as +-- "actionUrl", is omitted when empty, and Updatecli only ever fills it from an open pull +-- request. So "$.Actions.*.actionUrl" existing is an exact, self clearing marker for +-- "a pull request is open right now", and it is already true of every report stored so far. +-- +-- The expression must stay byte for byte the one in openActionSQLExpr, otherwise the queries +-- keep returning the right reports while silently falling back to a sequential scan. +-- +-- An expression index is used rather than a denormalized column: migration 000010 exists +-- precisely because a denormalized column silently drifted from the payload, and a generated +-- column would rewrite the whole table. An index needs no backfill, cannot drift, and covers +-- every existing row as soon as it is built. The result and the range predicates are part of +-- it so that the reports search and the reports summary, which always filter on a time range +-- and group per result, keep their index only scan. +BEGIN; + +CREATE INDEX IF NOT EXISTS idx_pipelinereports_updated_at_result_open_action +ON pipelineReports ( + updated_at, + pipeline_result, + (jsonb_path_exists(data, '$.Actions.*.actionUrl')) +); + +COMMIT; diff --git a/pkg/database/report.go b/pkg/database/report.go index 396d670f..e12e577b 100644 --- a/pkg/database/report.go +++ b/pkg/database/report.go @@ -102,6 +102,10 @@ type SearchLatestReportsParams struct { // Results restricts the search to the reports whose pipeline result is one of // them. An empty list does not filter anything out. Results []string + // OpenAction restricts the search to the reports which carry an open action, such as + // a pull request still waiting to be merged, or to the ones which do not. A nil value + // does not filter anything out. + OpenAction *bool } // SearchLatestReports searches the latest reports according some parameters. @@ -176,6 +180,7 @@ func SearchLatestReports(params SearchLatestReportsParams) ([]SearchLatestReport } applyResultFilter(&query, params.Results) + applyOpenActionFilter(&query, params.OpenAction) // Total counter query must be built before applying pagination // because it needs to count all the reports matching the query. @@ -372,6 +377,10 @@ type ReportSummaryParams struct { // Results restricts the summary to the reports whose pipeline result is one of // them. An empty list does not filter anything out. Results []string + // OpenAction restricts the summary to the reports which carry an open action, such as + // a pull request still waiting to be merged, or to the ones which do not. A nil value + // does not filter anything out. + OpenAction *bool } // ReportResultSummaryEntry contains the number of reports per result for a single time bucket. @@ -380,6 +389,15 @@ type ReportResultSummaryEntry struct { Date string `json:"date"` // Results contains the number of reports per Updatecli result for that bucket. Results map[string]int `json:"results"` + // OpenActions contains, for each Updatecli result, how many of the reports counted in + // Results also carry an open action, such as a pull request still waiting to be merged. + // It is a breakdown of Results, not an addition to it, so its counts are always lower + // than or equal to the matching ones in Results. + // + // The interesting one is the count reported under the success result: those pipelines + // ran fine and had nothing to change only because the change is already waiting in a + // pull request. + OpenActions map[string]int `json:"open_actions"` // Total is the number of reports for that bucket, all results combined. Total int `json:"total"` } @@ -417,6 +435,7 @@ func SearchReportsSummary(params ReportSummaryParams) ([]ReportResultSummaryEntr // pipeline_result is denormalized from data ->> 'Result' when the report is // inserted, grouping on it avoids parsing the jsonb document of every report. "pipeline_result", + openActionSQLExpr, "count(*)", ), sm.Where( @@ -424,6 +443,7 @@ func SearchReportsSummary(params ReportSummaryParams) ([]ReportResultSummaryEntr ), sm.GroupBy(dateTrunc), sm.GroupBy("pipeline_result"), + sm.GroupBy(openActionSQLExpr), sm.OrderBy(dateTrunc), ) @@ -432,6 +452,7 @@ func SearchReportsSummary(params ReportSummaryParams) ([]ReportResultSummaryEntr } applyResultFilter(&query, params.Results) + applyOpenActionFilter(&query, params.OpenAction) if len(params.Labels) > 0 { // The report window is widened to whole buckets so the label lookup must cover @@ -467,23 +488,31 @@ func SearchReportsSummary(params ReportSummaryParams) ([]ReportResultSummaryEntr defer rows.Close() countByDate := map[string]map[string]int{} + openActionCountByDate := map[string]map[string]int{} totalCount := 0 for rows.Next() { bucket := time.Time{} reportResult := "" + hasOpenAction := false count := 0 - if err := rows.Scan(&bucket, &reportResult, &count); err != nil { + if err := rows.Scan(&bucket, &reportResult, &hasOpenAction, &count); err != nil { return nil, 0, fmt.Errorf("parsing result: %s", err) } date := bucket.UTC().Format(summaryDateFormat) if countByDate[date] == nil { countByDate[date] = map[string]int{} + openActionCountByDate[date] = map[string]int{} } - countByDate[date][summaryResultKey(reportResult)] += count + resultKey := summaryResultKey(reportResult) + + countByDate[date][resultKey] += count + if hasOpenAction { + openActionCountByDate[date][resultKey] += count + } totalCount += count } @@ -494,12 +523,14 @@ func SearchReportsSummary(params ReportSummaryParams) ([]ReportResultSummaryEntr dataset := []ReportResultSummaryEntry{} for bucket := firstBucket; !bucket.After(lastBucket); bucket = nextBucket(bucket, granularity) { entry := ReportResultSummaryEntry{ - Date: bucket.Format(summaryDateFormat), - Results: map[string]int{}, + Date: bucket.Format(summaryDateFormat), + Results: map[string]int{}, + OpenActions: map[string]int{}, } for _, r := range summaryResultKeys { entry.Results[r] = 0 + entry.OpenActions[r] = 0 } for r, count := range countByDate[entry.Date] { @@ -507,6 +538,10 @@ func SearchReportsSummary(params ReportSummaryParams) ([]ReportResultSummaryEntr entry.Total += count } + for r, count := range openActionCountByDate[entry.Date] { + entry.OpenActions[r] += count + } + dataset = append(dataset, entry) } @@ -997,6 +1032,40 @@ func applyResultFilter(query *bob.BaseQuery[*dialect.SelectQuery], results []str query.Apply(sm.Where(psql.Quote("pipeline_result").In(args...))) } +// openActionSQLExpr is true of the reports carrying at least one action left open, which is +// how Updatecli reports a pull request still waiting to be merged. +// +// reports.Action.Link is serialized as "actionUrl" and omitted when empty, and Updatecli +// only ever fills it from an open pull request: CheckActionExist queries the forge for open +// pull requests only, and the pull request handler resets the link when it closes one. So +// the presence of that key is a self clearing marker, and it is already true of every report +// stored so far rather than only of the ones produced from now on. +// +// The jsonpath is inlined rather than bound as a parameter on purpose: an expression index +// only matches a literal expression, so binding it would cost +// idx_pipelinereports_updated_at_result_open_action. It contains no user input. +// +// It must also stay free of the jsonpath filter operator: bob reads "?" as a placeholder, +// so a path such as '$.Actions.*.actionUrl ? (@ != "")' silently consumes an argument and +// builds a query which matches nothing. Guarding against an empty link is unnecessary +// anyway, "actionUrl" is omitempty so it is absent rather than empty. +const openActionSQLExpr = `jsonb_path_exists(data, '$.Actions.*.actionUrl')` + +// applyOpenActionFilter restricts the given query to the reports which do, or which do not, +// carry an open action. A nil openAction does not filter anything out. +// +// This is deliberately a dimension of its own rather than a fifth pipeline result: an open +// action is orthogonal to the result. A pipeline may have succeeded because its change is +// already in an open pull request, but it may also have changed something and just opened +// one, or be failing while a pull request from a previous run is still around. +func applyOpenActionFilter(query *bob.BaseQuery[*dialect.SelectQuery], openAction *bool) { + if openAction == nil { + return + } + + query.Apply(sm.Where(psql.Raw(openActionSQLExpr+" = ?", psql.Arg(*openAction)))) +} + // applyScmFilter restricts the given query to the reports associated to a specific scm. // An empty scmID does not filter anything while "none", "null", or "nil" only keeps // the reports which are not associated to any scm. diff --git a/pkg/database/scm.go b/pkg/database/scm.go index 37f85b97..7384d1b7 100644 --- a/pkg/database/scm.go +++ b/pkg/database/scm.go @@ -136,6 +136,15 @@ type ScmSummaryData struct { TotalResult int `json:"total_result"` // TotalActionURLs is the total number of unique action URLs for this SCM. TotalActionURLs int `json:"total_action_urls"` + // TotalOpenActionByResult is a map of result types to the number of pipelines in that + // result which also carry an open action, such as a pull request still waiting to be + // merged. It is a breakdown of TotalResultByType, so its counts are always lower than + // or equal to the matching ones there. + // + // Unlike TotalActionURLs, which counts distinct action URLs, this counts pipelines: a + // single pull request grouping the changes of several pipelines is counted once there + // and once per pipeline here. + TotalOpenActionByResult map[string]int `json:"total_open_action_by_result"` } // SCMBranchDataset represents a map of branches and their summary data for a single SCM URL. @@ -153,7 +162,11 @@ type GetSCMSummaryParams struct { Labels map[string]string // Results restricts the summary to the reports whose pipeline result is one of // them. An empty list does not filter anything out. - Results []string + Results []string + // OpenAction restricts the summary to the pipelines which carry an open action, such as + // a pull request still waiting to be merged, or to the ones which do not. A nil value + // does not filter anything out. + OpenAction *bool TotalCount int TotalActions int Ctx context.Context @@ -214,6 +227,9 @@ func GetSCMSummary(params GetSCMSummaryParams) (*SCMDataset, error) { psql.Raw("data ->> 'ID'"), ), sm.With("filtered_reports").As(filteredSCMsQuery), + // The action URLs are read with the same jsonpath as openActionSQLExpr, so that + // a pipeline counted as carrying an open action here is the one the reports + // search and the reports summary would return too. sm.Columns("id", "data ->> 'Result'", "jsonb_path_query_array(data, '$.Actions.*.actionUrl')"), sm.From("filtered_reports"), sm.OrderBy(psql.Raw("data ->> 'ID'")), @@ -239,8 +255,9 @@ func GetSCMSummary(params GetSCMSummaryParams) (*SCMDataset, error) { } d := ScmSummaryData{ - ID: scmID.String(), - TotalResultByType: make(map[string]int), + ID: scmID.String(), + TotalResultByType: make(map[string]int), + TotalOpenActionByResult: make(map[string]int), } dataset.Data[scmURL][scmBranch] = d @@ -258,16 +275,22 @@ func GetSCMSummary(params GetSCMSummaryParams) (*SCMDataset, error) { return nil, fmt.Errorf("scanning scm summary row: %w", err) } - // The results are dropped here rather than in the query above on purpose. - // That query keeps the latest report of every pipeline, so this summary - // reports where each pipeline stands now; filtering the reports before - // that would instead keep the latest report which happened to carry one + hasOpenAction := len(actionUrls) > 0 + + // The results and the open actions are dropped here rather than in the query + // above on purpose. That query keeps the latest report of every pipeline, so + // this summary reports where each pipeline stands now; filtering the reports + // before that would instead keep the latest report which happened to carry one // of those results, reporting a pipeline as failing long after it // recovered. if len(params.Results) > 0 && !slices.Contains(params.Results, result) { continue } + if params.OpenAction != nil && *params.OpenAction != hasOpenAction { + continue + } + resultFound := false for r := range dataset.Data[scmURL][scmBranch].TotalResultByType { if r == result { @@ -280,6 +303,10 @@ func GetSCMSummary(params GetSCMSummaryParams) (*SCMDataset, error) { dataset.Data[scmURL][scmBranch].TotalResultByType[result] = 1 } + if hasOpenAction { + dataset.Data[scmURL][scmBranch].TotalOpenActionByResult[result]++ + } + for i := range actionUrls { if _, ok := isActionURLsFound[actionUrls[i]]; !ok { isActionURLsFound[actionUrls[i]] = true diff --git a/pkg/server/endpoints_test.go b/pkg/server/endpoints_test.go index b55a42ab..949060f6 100644 --- a/pkg/server/endpoints_test.go +++ b/pkg/server/endpoints_test.go @@ -9,6 +9,7 @@ import ( "maps" "net/http" "net/http/httptest" + "sort" "testing" "time" @@ -526,15 +527,20 @@ func TestEndpoints(t *testing.T) { } // bucketEntry builds the expected response entry for a bucket, starting from - // a zeroed set of results. + // a zeroed set of results. None of the reports seeded here carries an action, so + // the open action breakdown is always zeroed; it is covered on its own below. bucketEntry := func(date string, results map[string]any) map[string]any { - allResults := map[string]any{ - "✔": float64(0), - "✗": float64(0), - "⚠": float64(0), - "-": float64(0), - "unknown": float64(0), + zeroedResults := func() map[string]any { + return map[string]any{ + "✔": float64(0), + "✗": float64(0), + "⚠": float64(0), + "-": float64(0), + "unknown": float64(0), + } } + + allResults := zeroedResults() total := float64(0) for k, v := range results { allResults[k] = v @@ -542,9 +548,10 @@ func TestEndpoints(t *testing.T) { } return map[string]any{ - "date": date, - "results": allResults, - "total": total, + "date": date, + "results": allResults, + "open_actions": zeroedResults(), + "total": total, } } @@ -863,6 +870,225 @@ func TestEndpoints(t *testing.T) { }) }) }) + + t.Run("filtering on an action left open", func(t *testing.T) { + // Updatecli reports a pipeline which had nothing to change as a success even when + // the change it would have made is already waiting in an open pull request. That + // is the state which needs a human, yet the result alone cannot express it: the + // only thing telling it apart from a genuinely up to date pipeline is the action + // link the report carries. + truncateReports(t) + t.Cleanup(func() { + truncateReports(t) + }) + + seed := func(name, pipelineResult, actionURL string) string { + t.Helper() + + id, err := database.InsertReport(ctx, reports.Report{ + Name: name, + Result: pipelineResult, + ID: name, + PipelineID: "venom", + Actions: map[string]*reports.Action{ + "default": {ID: "default", Link: actionURL}, + }, + }) + require.NoError(t, err) + + return id + } + + successWithOpenPR := seed("succeeded, pull request still open", "✔", + "https://example.com/testing/pull/42") + seed("succeeded, nothing to follow up", "✔", "") + attentionWithOpenPR := seed("changed something and opened a pull request", "⚠", + "https://example.com/testing/pull/43") + + // reportNames returns the name of every report of a search response, which + // identifies the seeded reports more readably than their database id. + reportNames := func(resp *http.Response) []string { + t.Helper() + + blob := struct { + Data []struct { + Name string + } + TotalCount int `json:"total_count"` + }{} + require.NoError(t, json.NewDecoder(resp.Body).Decode(&blob)) + defer resp.Body.Close() + + names := make([]string, 0, len(blob.Data)) + for _, report := range blob.Data { + names = append(names, report.Name) + } + + // A filter dropping reports from the page while still counting them in the + // total breaks pagination, so the two are checked against each other. + assert.Equal(t, len(names), blob.TotalCount) + sort.Strings(names) + + return names + } + + t.Run("POST /api/pipeline/reports/search", func(t *testing.T) { + t.Run("without the filter", func(t *testing.T) { + resp := doPostRequest(t, srv, "/api/pipeline/reports/search", map[string]any{}) + + assert.Equal(t, []string{ + "changed something and opened a pull request", + "succeeded, nothing to follow up", + "succeeded, pull request still open", + }, reportNames(resp)) + }) + + t.Run("with an open action", func(t *testing.T) { + resp := doPostRequest(t, srv, "/api/pipeline/reports/search", map[string]any{ + "open_action": true, + }) + + assert.Equal(t, []string{ + "changed something and opened a pull request", + "succeeded, pull request still open", + }, reportNames(resp)) + }) + + t.Run("without any open action", func(t *testing.T) { + resp := doPostRequest(t, srv, "/api/pipeline/reports/search", map[string]any{ + "open_action": false, + }) + + assert.Equal(t, []string{"succeeded, nothing to follow up"}, reportNames(resp)) + }) + + t.Run("combined with a result", func(t *testing.T) { + // This is the combination the whole dimension exists for: the pipelines + // which succeeded only because their change is already waiting in a pull + // request nobody merged. + resp := doPostRequest(t, srv, "/api/pipeline/reports/search", map[string]any{ + "results": []string{"✔"}, + "open_action": true, + }) + + assert.Equal(t, []string{"succeeded, pull request still open"}, reportNames(resp)) + }) + }) + + t.Run("POST /api/pipeline/reports/summary", func(t *testing.T) { + summaryOf := func(body map[string]any) (results, openActions map[string]any, totalCount float64) { + t.Helper() + + blob := map[string]any{} + resp := doPostRequest(t, srv, "/api/pipeline/reports/summary", body) + require.NoError(t, json.NewDecoder(resp.Body).Decode(&blob)) + defer resp.Body.Close() + + data := blob["data"].([]any) + today := data[len(data)-1].(map[string]any) + + return today["results"].(map[string]any), + today["open_actions"].(map[string]any), + blob["total_count"].(float64) + } + + t.Run("reports the open actions as a breakdown of the results", func(t *testing.T) { + // Nothing is filtered out here: the breakdown is what lets a dashboard + // split the success bucket without having to run a second query. + results, openActions, totalCount := summaryOf(map[string]any{"days": 1}) + + assert.Equal(t, float64(3), totalCount) + assert.Equal(t, float64(2), results["✔"]) + assert.Equal(t, float64(1), results["⚠"]) + assert.Equal(t, float64(1), openActions["✔"]) + assert.Equal(t, float64(1), openActions["⚠"]) + assert.Equal(t, float64(0), openActions["✗"]) + }) + + t.Run("filtered on an open action", func(t *testing.T) { + results, openActions, totalCount := summaryOf(map[string]any{ + "days": 1, + "open_action": true, + }) + + assert.Equal(t, float64(2), totalCount) + assert.Equal(t, float64(1), results["✔"]) + assert.Equal(t, float64(1), openActions["✔"]) + }) + + t.Run("filtered on the absence of an open action", func(t *testing.T) { + results, openActions, totalCount := summaryOf(map[string]any{ + "days": 1, + "open_action": false, + }) + + assert.Equal(t, float64(1), totalCount) + assert.Equal(t, float64(1), results["✔"]) + assert.Equal(t, float64(0), openActions["✔"]) + }) + }) + + t.Run("POST /api/pipeline/scms/search", func(t *testing.T) { + scmID, err := database.InsertSCM(ctx, "https://example.com/openaction.git", "main") + require.NoError(t, err) + t.Cleanup(func() { + deleteSCM(t, scmID) + }) + + attachReportToSCM(t, successWithOpenPR, scmID) + attachReportToSCM(t, attentionWithOpenPR, scmID) + + branchOf := func(body map[string]any) map[string]any { + t.Helper() + + blob := map[string]any{} + resp := doPostRequest(t, srv, "/api/pipeline/scms/search", body) + require.NoError(t, json.NewDecoder(resp.Body).Decode(&blob)) + defer resp.Body.Close() + + data := blob["data"].(map[string]any) + repository := data["https://example.com/openaction.git"].(map[string]any) + + return repository["main"].(map[string]any) + } + + t.Run("breaks the open actions down per result", func(t *testing.T) { + branch := branchOf(map[string]any{"summary": true, "scmid": scmID}) + + assert.Equal(t, map[string]any{"✔": float64(1), "⚠": float64(1)}, + branch["total_result_by_type"]) + assert.Equal(t, map[string]any{"✔": float64(1), "⚠": float64(1)}, + branch["total_open_action_by_result"]) + // Two pipelines, but each on a pull request of its own. + assert.Equal(t, float64(2), branch["total_action_urls"]) + }) + + t.Run("filtered on a result and an open action", func(t *testing.T) { + branch := branchOf(map[string]any{ + "summary": true, + "scmid": scmID, + "results": []string{"✔"}, + "open_action": true, + }) + + assert.Equal(t, map[string]any{"✔": float64(1)}, branch["total_result_by_type"]) + assert.Equal(t, map[string]any{"✔": float64(1)}, branch["total_open_action_by_result"]) + }) + + t.Run("filtered on the absence of an open action", func(t *testing.T) { + // Both reports attached to this scm carry one, so the summary keeps the + // branch but empties its counts. + branch := branchOf(map[string]any{ + "summary": true, + "scmid": scmID, + "open_action": false, + }) + + assert.Equal(t, map[string]any{}, branch["total_result_by_type"]) + assert.Equal(t, map[string]any{}, branch["total_open_action_by_result"]) + }) + }) + }) } // hourStart returns the beginning of the UTC hour of the provided time. diff --git a/pkg/server/report_handlers.go b/pkg/server/report_handlers.go index 18355b81..f0ab216e 100644 --- a/pkg/server/report_handlers.go +++ b/pkg/server/report_handlers.go @@ -135,6 +135,15 @@ func SearchPipelineReports(c *gin.Context) { // "✔", "✗", "⚠" or "-". A report matches when its result is any of them. // This is optional and an empty list does not filter anything out. Results []string `json:"results,omitempty"` + // OpenAction filters reports by whether they carry an action left open, such as a + // pull request still waiting to be merged. This is optional: unset does not filter + // anything out, true only keeps the reports with an open action and false only the + // ones without. + // + // Combined with results it isolates the pipelines which succeeded because their + // change is already waiting in a pull request, which a result alone cannot express: + // {"results": ["✔"], "open_action": true}. + OpenAction *bool `json:"open_action,omitempty"` } queryParams := queryData{} @@ -162,6 +171,7 @@ func SearchPipelineReports(c *gin.Context) { Latest: queryParams.Latest, Labels: queryParams.Labels, Results: queryParams.Results, + OpenAction: queryParams.OpenAction, }, ) if err != nil { @@ -202,6 +212,14 @@ type SearchPipelineReportsSummaryRequest struct { // "✔", "✗", "⚠" or "-". A report is counted when its result is any of them. // An empty list does not filter anything out. Results []string `json:"results,omitempty"` + // OpenAction filters reports by whether they carry an action left open, such as a + // pull request still waiting to be merged. This is optional: unset does not filter + // anything out, true only counts the reports with an open action and false only the + // ones without. + // + // The same breakdown is reported without filtering anything out under the open_actions + // key of every bucket. + OpenAction *bool `json:"open_action,omitempty"` // StartTime is the start time for the time range filter. // Time format is: 2006-01-02 15:04:05Z07:00 StartTime string `json:"start_time,omitempty"` @@ -323,6 +341,7 @@ func SearchPipelineReportsSummary(c *gin.Context) { ScmID: queryParams.ScmID, Labels: queryParams.Labels, Results: queryParams.Results, + OpenAction: queryParams.OpenAction, StartTime: queryParams.StartTime, EndTime: queryParams.EndTime, }, diff --git a/pkg/server/scmdb_handlers.go b/pkg/server/scmdb_handlers.go index 9dedf57a..1854482b 100644 --- a/pkg/server/scmdb_handlers.go +++ b/pkg/server/scmdb_handlers.go @@ -33,6 +33,11 @@ type SearchSCMsRequest struct { // Results filters SCM summaries by pipeline result, such as "✔", "✗", "⚠" or // "-". An empty list does not filter anything out. Results []string `json:"results,omitempty"` + // OpenAction filters SCM summaries by whether a pipeline carries an action left open, + // such as a pull request still waiting to be merged. This is optional: unset does not + // filter anything out, true only keeps the pipelines with an open action and false only + // the ones without. + OpenAction *bool `json:"open_action,omitempty"` // URL is the SCM URL to filter by. URL string `json:"url,omitempty"` // Branch is the SCM branch to filter by. @@ -78,15 +83,15 @@ func SearchSCMs(c *gin.Context) { } if queryParams.Summary { - findSCMSummary( - c, - rows, - totalCount, - queryParams.StartTime, - queryParams.EndTime, - queryParams.Labels, - queryParams.Results, - ) + findSCMSummary(c, findSCMSummaryParams{ + ScmRows: rows, + TotalCount: totalCount, + StartTime: queryParams.StartTime, + EndTime: queryParams.EndTime, + Labels: queryParams.Labels, + Results: queryParams.Results, + OpenAction: queryParams.OpenAction, + }) return } @@ -158,7 +163,13 @@ func ListSCMs(c *gin.Context) { } if summary { - findSCMSummary(c, rows, totalCount, queryValues.Get("start_time"), queryValues.Get("end_time"), map[string]string{}, nil) + findSCMSummary(c, findSCMSummaryParams{ + ScmRows: rows, + TotalCount: totalCount, + StartTime: queryValues.Get("start_time"), + EndTime: queryValues.Get("end_time"), + Labels: map[string]string{}, + }) return } @@ -185,19 +196,39 @@ type FindSCMSummaryResponse struct { Data map[string]database.SCMBranchDataset `json:"data"` } +// findSCMSummaryParams contains the filters applied to a git repositories summary. +type findSCMSummaryParams struct { + // ScmRows are the SCMs to summarize. + ScmRows []model.SCM + // TotalCount is the number of SCMs matching the search, before pagination. + TotalCount int + // StartTime and EndTime define the time range the reports are summarized over. + StartTime string + EndTime string + // Labels restricts the summary to the reports matching those labels. + Labels map[string]string + // Results restricts the summary to the pipelines whose result is one of them. An empty + // list does not filter anything out. + Results []string + // OpenAction restricts the summary to the pipelines which carry an open action, or to + // the ones which do not. A nil value does not filter anything out. + OpenAction *bool +} + // findSCMSummary returns a summary of all git repositories detected. -func findSCMSummary(c *gin.Context, scmRows []model.SCM, totalCount int, startTime, endTime string, labels map[string]string, results []string) { +func findSCMSummary(c *gin.Context, params findSCMSummaryParams) { var data map[string]database.SCMBranchDataset dataset, err := database.GetSCMSummary(database.GetSCMSummaryParams{ Ctx: c, - ScmRows: scmRows, - TotalCount: totalCount, + ScmRows: params.ScmRows, + TotalCount: params.TotalCount, MonitoringDurationDays: monitoringDurationDays, - StartTime: startTime, - EndTime: endTime, - Labels: labels, - Results: results, + StartTime: params.StartTime, + EndTime: params.EndTime, + Labels: params.Labels, + Results: params.Results, + OpenAction: params.OpenAction, }) if err != nil { logrus.Errorf("getting scm summary failed: %s", err) @@ -213,6 +244,6 @@ func findSCMSummary(c *gin.Context, scmRows []model.SCM, totalCount int, startTi c.JSON(http.StatusOK, FindSCMSummaryResponse{ Data: data, - TotalCount: totalCount, + TotalCount: params.TotalCount, }) }