Add database filters to gpbackman commands.

end_to_end/end_to_end_suite_test.go

@@ -78,6 +78,7 @@ const (
 	TOTAL_RELATIONS               = 37
 	TOTAL_RELATIONS_AFTER_EXCLUDE = 21
 	TOTAL_CREATE_STATEMENTS       = 9
+	gpbackmanFilterDatabase       = "gpbackman_filter_db"
 )
 
 // This function is run automatically by ginkgo before any tests are run.
@@ -90,14 +91,18 @@ func init() {
  * to allow checking its output.
  */
 func gpbackup(gpbackupPath string, backupHelperPath string, args ...string) []byte {
-	return runGpbackup(gpbackupPath, backupHelperPath, true, args...)
+	return gpbackupForDatabase("testdb", gpbackupPath, backupHelperPath, args...)
 }
 
 func gpbackupWithHistoryStandbySync(gpbackupPath string, backupHelperPath string, args ...string) []byte {
-	return runGpbackup(gpbackupPath, backupHelperPath, false, args...)
+	return runGpbackupForDatabase("testdb", gpbackupPath, backupHelperPath, false, args...)
 }
 
-func runGpbackup(gpbackupPath string, backupHelperPath string, disableHistoryStandbySync bool, args ...string) []byte {
+func gpbackupForDatabase(databaseName string, gpbackupPath string, backupHelperPath string, args ...string) []byte {
+	return runGpbackupForDatabase(databaseName, gpbackupPath, backupHelperPath, true, args...)
+}
+
+func runGpbackupForDatabase(databaseName string, gpbackupPath string, backupHelperPath string, disableHistoryStandbySync bool, args ...string) []byte {
 	if useOldBackupVersion {
 		_ = os.Chdir("..")
 		command := exec.Command("make", "install", fmt.Sprintf("helper_path=%s", backupHelperPath))
@@ -107,7 +112,7 @@ func runGpbackup(gpbackupPath string, backupHelperPath string, disableHistorySta
 	if disableHistoryStandbySync && !useOldBackupVersion && !hasCommandArgument(args, "--no-history-sync-standby") {
 		args = append(args, "--no-history-sync-standby")
 	}
-	args = append([]string{"--verbose", "--dbname", "testdb"}, args...)
+	args = append([]string{"--verbose", "--dbname", databaseName}, args...)
 	command := exec.Command(gpbackupPath, args...)
 	return mustRunCommand(command)
 }
@@ -744,6 +749,23 @@ func end_to_end_teardown() {
 	_ = os.RemoveAll(backupDir)
 }
 
+func setupGpbackmanFilterDatabase() {
+	_ = exec.Command("dropdb", gpbackmanFilterDatabase).Run()
+	Expect(exec.Command("createdb", gpbackmanFilterDatabase).Run()).To(Succeed())
+
+	filterConn := testutils.SetupTestDbConn(gpbackmanFilterDatabase)
+	defer filterConn.Close()
+	testhelper.AssertQueryRuns(filterConn, `
+		CREATE TABLE public.e2e_data (id integer, value text) DISTRIBUTED BY (id);
+		INSERT INTO public.e2e_data (id, value)
+		SELECT i, 'e2e-value-' || i FROM generate_series(1, 100) AS i;
+	`)
+}
+
+func teardownGpbackmanFilterDatabase() {
+	_ = exec.Command("dropdb", gpbackmanFilterDatabase).Run()
+}
+
 var _ = Describe("backup and restore end to end tests", func() {
 	BeforeEach(func() {
 		end_to_end_setup()

end_to_end/gpbackman_test.go

@@ -76,6 +76,7 @@ var _ = Describe("gpbackman end to end tests", func() {
 
 		BeforeEach(func() {
 			end_to_end_setup()
+			setupGpbackmanFilterDatabase()
 			historyDB = getHistoryDBPathForCluster()
 			timestampMap = make(map[string]string)
 
@@ -109,9 +110,15 @@ var _ = Describe("gpbackman end to end tests", func() {
 				"--backup-dir", backupDir,
 				"--metadata-only")
 			timestampMap["metadata_only"] = getBackupTimestamp(string(output))
+
+			output = gpbackupForDatabase(gpbackmanFilterDatabase, gpbackupPath, backupHelperPath,
+				"--backup-dir", backupDir,
+				"--include-table", "public.e2e_data")
+			timestampMap["filter_database"] = getBackupTimestamp(string(output))
 		})
 
 		AfterEach(func() {
+			teardownGpbackmanFilterDatabase()
 			end_to_end_teardown()
 		})
 
@@ -121,9 +128,35 @@ var _ = Describe("gpbackman end to end tests", func() {
 				"--history-db", historyDB,
 			)
 			lines := countBackupInfoLines(output)
-			Expect(lines).To(BeNumerically(">=", 5),
-				fmt.Sprintf("Expected at least 5 backup entries, got %d.\nOutput:\n%s",
+			Expect(lines).To(BeNumerically(">=", 6),
+				fmt.Sprintf("Expected at least 6 backup entries, got %d.\nOutput:\n%s",
 					lines, string(output)))
+			Expect(string(output)).To(ContainSubstring("testdb"))
+			Expect(string(output)).To(ContainSubstring(gpbackmanFilterDatabase))
+		})
+
+		It("filters by database and composes with detail and table filters", func() {
+			output := gpbackman(
+				"backup-info",
+				"--history-db", historyDB,
+				"--database", gpbackmanFilterDatabase,
+				"--type", "full",
+				"--table", "public.e2e_data",
+				"--detail",
+			)
+			Expect(countBackupInfoLines(output)).To(Equal(1))
+			Expect(string(output)).To(ContainSubstring(timestampMap["filter_database"]))
+			Expect(string(output)).To(ContainSubstring(gpbackmanFilterDatabase))
+			Expect(string(output)).To(ContainSubstring("e2e_data"))
+		})
+
+		It("returns no rows for an unknown database", func() {
+			output := gpbackman(
+				"backup-info",
+				"--history-db", historyDB,
+				"--database", "gpbackman_unknown_db",
+			)
+			Expect(countBackupInfoLines(output)).To(Equal(0))
 		})
 
 		It("filters by type full", func() {
@@ -226,6 +259,17 @@ var _ = Describe("gpbackman end to end tests", func() {
 			Expect(err).To(HaveOccurred())
 		})
 
+		It("rejects incompatible flags --timestamp with --database", func() {
+			_, err := gpbackmanWithError(
+				"backup-info",
+				"--history-db", historyDB,
+				"--timestamp", timestampMap["filter_database"],
+				"--database", gpbackmanFilterDatabase,
+				"--detail",
+			)
+			Expect(err).To(HaveOccurred())
+		})
+
 		It("rejects invalid timestamp format", func() {
 			_, err := gpbackmanWithError(
 				"backup-info",
@@ -614,6 +658,32 @@ var _ = Describe("gpbackman end to end tests", func() {
 			)
 			// Success if no error was thrown
 		})
+
+		It("cleans only the selected database's eligible backups", func() {
+			setupGpbackmanFilterDatabase()
+			DeferCleanup(teardownGpbackmanFilterDatabase)
+
+			primaryOutput := gpbackup(gpbackupPath, backupHelperPath,
+				"--backup-dir", backupDir)
+			primaryTimestamp := getBackupTimestamp(string(primaryOutput))
+			filterOutput := gpbackupForDatabase(gpbackmanFilterDatabase, gpbackupPath, backupHelperPath,
+				"--backup-dir", backupDir)
+			filterTimestamp := getBackupTimestamp(string(filterOutput))
+
+			gpbackman(
+				"backup-clean",
+				"--history-db", historyDB,
+				"--before-timestamp", "99991231235959",
+				"--database", gpbackmanFilterDatabase,
+			)
+
+			primaryActive := queryHistoryDB(historyDB,
+				fmt.Sprintf("SELECT count(*) FROM backups WHERE timestamp = '%s' AND date_deleted = ''", primaryTimestamp))
+			Expect(primaryActive).To(Equal("1"), "backup for the default database should remain active")
+			filterDeleted := queryHistoryDB(historyDB,
+				fmt.Sprintf("SELECT count(*) FROM backups WHERE timestamp = '%s' AND date_deleted != ''", filterTimestamp))
+			Expect(filterDeleted).To(Equal("1"), "backup for the selected database should be deleted")
+		})
 	})
 
 	// ------------------------------------------------------------------ //
@@ -716,6 +786,46 @@ var _ = Describe("gpbackman end to end tests", func() {
 			Expect(count2).To(Equal("1"),
 				"Non-deleted backup should remain in history")
 		})
+
+		It("cleans deleted history and related rows only for the selected database", func() {
+			setupGpbackmanFilterDatabase()
+			DeferCleanup(teardownGpbackmanFilterDatabase)
+
+			primaryOutput := gpbackup(gpbackupPath, backupHelperPath,
+				"--backup-dir", backupDir)
+			primaryTimestamp := getBackupTimestamp(string(primaryOutput))
+			filterOutput := gpbackupForDatabase(gpbackmanFilterDatabase, gpbackupPath, backupHelperPath,
+				"--backup-dir", backupDir,
+				"--include-table", "public.e2e_data")
+			filterTimestamp := getBackupTimestamp(string(filterOutput))
+
+			gpbackman(
+				"backup-delete",
+				"--history-db", historyDB,
+				"--timestamp", primaryTimestamp,
+			)
+			gpbackman(
+				"backup-delete",
+				"--history-db", historyDB,
+				"--timestamp", filterTimestamp,
+			)
+			Expect(queryHistoryDB(historyDB,
+				fmt.Sprintf("SELECT count(*) FROM include_relations WHERE timestamp = '%s'", filterTimestamp))).To(Equal("1"))
+
+			gpbackman(
+				"history-clean",
+				"--history-db", historyDB,
+				"--before-timestamp", "99991231235959",
+				"--database", gpbackmanFilterDatabase,
+			)
+
+			Expect(queryHistoryDB(historyDB,
+				fmt.Sprintf("SELECT count(*) FROM backups WHERE timestamp = '%s'", primaryTimestamp))).To(Equal("1"))
+			Expect(queryHistoryDB(historyDB,
+				fmt.Sprintf("SELECT count(*) FROM backups WHERE timestamp = '%s'", filterTimestamp))).To(Equal("0"))
+			Expect(queryHistoryDB(historyDB,
+				fmt.Sprintf("SELECT count(*) FROM include_relations WHERE timestamp = '%s'", filterTimestamp))).To(Equal("0"))
+		})
 	})
 
 	// ------------------------------------------------------------------ //

gpbackman/COMMANDS.md

@@ -65,6 +65,11 @@ To delete backup sets older than the given number of days, use the --older-than-
 To delete backup sets newer than the given timestamp, use the --after-timestamp option.
 Only --older-than-days, --before-timestamp or --after-timestamp option must be specified.
 
+Use --database to clean backup sets only for the specified database. Without --database,
+cleanup includes backup sets for all databases in the history database.
+Database names are matched exactly and case-sensitively against backup history.
+For database names that require quoting, include the double quotes in the --database value.
+
 By default, the existence of dependent backups is checked and deletion process is not performed,
 unless the --cascade option is passed in.
 
@@ -99,6 +104,7 @@ Flags:
       --backup-dir string         the full path to backup directory for local backups
       --before-timestamp string   delete backup sets older than the given timestamp
       --cascade                   delete all dependent backups
+      --database string           delete backup sets only for the specified database
   -h, --help                      help for backup-clean
       --history-sync-standby-timeout int   shared rsync and remote install timeout in seconds; must be an integer between 1 and 86400 (default 300)
       --no-history-sync-standby   skip automatic gpbackup_history.db sync to standby coordinator after this command
@@ -124,6 +130,20 @@ Delete backups older than a timestamp:
   --cascade
 ```
 
+Delete local backups only for database `analytics`:
+```bash
+./gpbackman backup-clean \
+  --before-timestamp 20240701100000 \
+  --database analytics
+```
+
+For database `Sales DB`, include double quotes in the flag value:
+```bash
+./gpbackman backup-clean \
+  --before-timestamp 20240701100000 \
+  --database '"Sales DB"'
+```
+
 Delete backups older than a number of days with multiple parallel processes:
 ```bash
 ./gpbackman backup-clean \
@@ -251,6 +271,12 @@ To display all backups, use --deleted and --failed options together.
 
 To display backups of a specific type, use the --type option.
 
+Without the --database option, backups for all databases are displayed.
+To display backups only for a specific database, use the --database option.
+Database names are matched exactly and case-sensitively against backup history.
+The --database value is used without transformation. For database names that require quoting,
+include the double quotes in the flag value, for example: --database '"Sales DB"'.
+
 To display backups that include the specified table, use the --table option. 
 The formatting rules for <schema>.<table> match those of the --include-table option in gpbackup.
 
@@ -273,7 +299,7 @@ To display a backup chain for a specific backup, use the --timestamp option.
 In this mode, the backup with the specified timestamp and all of its dependent backups will be displayed.
 The deleted and failed backups are always included in this mode.
 To display object filtering details in this mode, use the --detail option.
-When --timestamp is set, the following options cannot be used: --type, --table, --schema, --exclude, --failed, --deleted.
+When --timestamp is set, the following options cannot be used: --database, --type, --table, --schema, --exclude, --failed, --deleted.
 
 To display the "object filtering details" column for all backups without using --timestamp, use the --detail option.
 
@@ -285,6 +311,7 @@ Usage:
   gpbackman backup-info [flags]
 
 Flags:
+      --database string    show backups only for the specified database (exact, case-sensitive match)
       --deleted            show deleted backups
       --detail             show object filtering details
       --exclude            show backups that exclude the specific table (format <schema>.<table>) or schema
@@ -378,6 +405,19 @@ Display info for active full backups from `gpbackup_history.db`:
  20230523101115 | Tue May 23 2023 10:11:15 | Success | demo     | full | include-schema   | gpbackup_s3_plugin | 01:01:00 |              
 ```
 
+Display active backups only for database `analytics`:
+```bash
+./gpbackman backup-info \
+  --database analytics
+```
+
+Display active full backups only for database `analytics`:
+```bash
+./gpbackman backup-info \
+  --database analytics \
+  --type full
+```
+
 Find all backups, including deleted ones, containing the `test1` schema.
 ```bash
 ./gpbackman backup-info \
@@ -477,6 +517,11 @@ To delete information about backups older than the given timestamp, use the --be
 To delete information about backups older than the given number of days, use the --older-than-day option. 
 Only --older-than-days or --before-timestamp option must be specified, not both.
 
+Use --database to clean history only for the specified database. Without --database,
+cleanup includes deleted backup history for all databases in the history database.
+Database names are matched exactly and case-sensitively against backup history.
+For database names that require quoting, include the double quotes in the --database value.
+
 The gpbackup_history.db file location can be set using the --history-db option.
 Can be specified only once. The full path to the file is required.
 If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY instead, pass the --auto-load-history-db flag.
@@ -486,6 +531,7 @@ Usage:
 
 Flags:
       --before-timestamp string   delete information about backups older than the given timestamp
+      --database string           delete backup history only for the specified database
   -h, --help                      help for history-clean
       --history-sync-standby-timeout int   shared rsync and remote install timeout in seconds; must be an integer between 1 and 86400 (default 300)
       --no-history-sync-standby   skip automatic gpbackup_history.db sync to standby coordinator after this command
@@ -514,6 +560,20 @@ Delete information about deleted backups from history database older than timest
   --before-timestamp 20240101100000
 ```
 
+Delete deleted backup history only for database `analytics`:
+```bash
+./gpbackman history-clean \
+  --before-timestamp 20240101100000 \
+  --database analytics
+```
+
+For database `Sales DB`, include double quotes in the flag value:
+```bash
+./gpbackman history-clean \
+  --before-timestamp 20240101100000 \
+  --database '"Sales DB"'
+```
+
 # Sync the history database to the standby coordinator (`history-sync`)
 
 Available options for `history-sync` command and their description: