diff --git a/end_to_end/end_to_end_suite_test.go b/end_to_end/end_to_end_suite_test.go index 75d60a32..4986c3c4 100644 --- a/end_to_end/end_to_end_suite_test.go +++ b/end_to_end/end_to_end_suite_test.go @@ -78,6 +78,7 @@ const ( TOTAL_RELATIONS = 37 TOTAL_RELATIONS_AFTER_EXCLUDE = 21 TOTAL_CREATE_STATEMENTS = 9 + gpbackmanFilterDatabase = "gpbackman_filter_db" ) // This function is run automatically by ginkgo before any tests are run. @@ -90,14 +91,18 @@ func init() { * to allow checking its output. */ func gpbackup(gpbackupPath string, backupHelperPath string, args ...string) []byte { - return runGpbackup(gpbackupPath, backupHelperPath, true, args...) + return gpbackupForDatabase("testdb", gpbackupPath, backupHelperPath, args...) } func gpbackupWithHistoryStandbySync(gpbackupPath string, backupHelperPath string, args ...string) []byte { - return runGpbackup(gpbackupPath, backupHelperPath, false, args...) + return runGpbackupForDatabase("testdb", gpbackupPath, backupHelperPath, false, args...) } -func runGpbackup(gpbackupPath string, backupHelperPath string, disableHistoryStandbySync bool, args ...string) []byte { +func gpbackupForDatabase(databaseName string, gpbackupPath string, backupHelperPath string, args ...string) []byte { + return runGpbackupForDatabase(databaseName, gpbackupPath, backupHelperPath, true, args...) +} + +func runGpbackupForDatabase(databaseName string, gpbackupPath string, backupHelperPath string, disableHistoryStandbySync bool, args ...string) []byte { if useOldBackupVersion { _ = os.Chdir("..") command := exec.Command("make", "install", fmt.Sprintf("helper_path=%s", backupHelperPath)) @@ -107,7 +112,7 @@ func runGpbackup(gpbackupPath string, backupHelperPath string, disableHistorySta if disableHistoryStandbySync && !useOldBackupVersion && !hasCommandArgument(args, "--no-history-sync-standby") { args = append(args, "--no-history-sync-standby") } - args = append([]string{"--verbose", "--dbname", "testdb"}, args...) + args = append([]string{"--verbose", "--dbname", databaseName}, args...) command := exec.Command(gpbackupPath, args...) return mustRunCommand(command) } @@ -744,6 +749,23 @@ func end_to_end_teardown() { _ = os.RemoveAll(backupDir) } +func setupGpbackmanFilterDatabase() { + _ = exec.Command("dropdb", gpbackmanFilterDatabase).Run() + Expect(exec.Command("createdb", gpbackmanFilterDatabase).Run()).To(Succeed()) + + filterConn := testutils.SetupTestDbConn(gpbackmanFilterDatabase) + defer filterConn.Close() + testhelper.AssertQueryRuns(filterConn, ` + CREATE TABLE public.e2e_data (id integer, value text) DISTRIBUTED BY (id); + INSERT INTO public.e2e_data (id, value) + SELECT i, 'e2e-value-' || i FROM generate_series(1, 100) AS i; + `) +} + +func teardownGpbackmanFilterDatabase() { + _ = exec.Command("dropdb", gpbackmanFilterDatabase).Run() +} + var _ = Describe("backup and restore end to end tests", func() { BeforeEach(func() { end_to_end_setup() diff --git a/end_to_end/gpbackman_test.go b/end_to_end/gpbackman_test.go index cabf46c1..f4ce0781 100644 --- a/end_to_end/gpbackman_test.go +++ b/end_to_end/gpbackman_test.go @@ -76,6 +76,7 @@ var _ = Describe("gpbackman end to end tests", func() { BeforeEach(func() { end_to_end_setup() + setupGpbackmanFilterDatabase() historyDB = getHistoryDBPathForCluster() timestampMap = make(map[string]string) @@ -109,9 +110,15 @@ var _ = Describe("gpbackman end to end tests", func() { "--backup-dir", backupDir, "--metadata-only") timestampMap["metadata_only"] = getBackupTimestamp(string(output)) + + output = gpbackupForDatabase(gpbackmanFilterDatabase, gpbackupPath, backupHelperPath, + "--backup-dir", backupDir, + "--include-table", "public.e2e_data") + timestampMap["filter_database"] = getBackupTimestamp(string(output)) }) AfterEach(func() { + teardownGpbackmanFilterDatabase() end_to_end_teardown() }) @@ -121,9 +128,35 @@ var _ = Describe("gpbackman end to end tests", func() { "--history-db", historyDB, ) lines := countBackupInfoLines(output) - Expect(lines).To(BeNumerically(">=", 5), - fmt.Sprintf("Expected at least 5 backup entries, got %d.\nOutput:\n%s", + Expect(lines).To(BeNumerically(">=", 6), + fmt.Sprintf("Expected at least 6 backup entries, got %d.\nOutput:\n%s", lines, string(output))) + Expect(string(output)).To(ContainSubstring("testdb")) + Expect(string(output)).To(ContainSubstring(gpbackmanFilterDatabase)) + }) + + It("filters by database and composes with detail and table filters", func() { + output := gpbackman( + "backup-info", + "--history-db", historyDB, + "--database", gpbackmanFilterDatabase, + "--type", "full", + "--table", "public.e2e_data", + "--detail", + ) + Expect(countBackupInfoLines(output)).To(Equal(1)) + Expect(string(output)).To(ContainSubstring(timestampMap["filter_database"])) + Expect(string(output)).To(ContainSubstring(gpbackmanFilterDatabase)) + Expect(string(output)).To(ContainSubstring("e2e_data")) + }) + + It("returns no rows for an unknown database", func() { + output := gpbackman( + "backup-info", + "--history-db", historyDB, + "--database", "gpbackman_unknown_db", + ) + Expect(countBackupInfoLines(output)).To(Equal(0)) }) It("filters by type full", func() { @@ -226,6 +259,17 @@ var _ = Describe("gpbackman end to end tests", func() { Expect(err).To(HaveOccurred()) }) + It("rejects incompatible flags --timestamp with --database", func() { + _, err := gpbackmanWithError( + "backup-info", + "--history-db", historyDB, + "--timestamp", timestampMap["filter_database"], + "--database", gpbackmanFilterDatabase, + "--detail", + ) + Expect(err).To(HaveOccurred()) + }) + It("rejects invalid timestamp format", func() { _, err := gpbackmanWithError( "backup-info", @@ -614,6 +658,32 @@ var _ = Describe("gpbackman end to end tests", func() { ) // Success if no error was thrown }) + + It("cleans only the selected database's eligible backups", func() { + setupGpbackmanFilterDatabase() + DeferCleanup(teardownGpbackmanFilterDatabase) + + primaryOutput := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + primaryTimestamp := getBackupTimestamp(string(primaryOutput)) + filterOutput := gpbackupForDatabase(gpbackmanFilterDatabase, gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + filterTimestamp := getBackupTimestamp(string(filterOutput)) + + gpbackman( + "backup-clean", + "--history-db", historyDB, + "--before-timestamp", "99991231235959", + "--database", gpbackmanFilterDatabase, + ) + + primaryActive := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT count(*) FROM backups WHERE timestamp = '%s' AND date_deleted = ''", primaryTimestamp)) + Expect(primaryActive).To(Equal("1"), "backup for the default database should remain active") + filterDeleted := queryHistoryDB(historyDB, + fmt.Sprintf("SELECT count(*) FROM backups WHERE timestamp = '%s' AND date_deleted != ''", filterTimestamp)) + Expect(filterDeleted).To(Equal("1"), "backup for the selected database should be deleted") + }) }) // ------------------------------------------------------------------ // @@ -716,6 +786,46 @@ var _ = Describe("gpbackman end to end tests", func() { Expect(count2).To(Equal("1"), "Non-deleted backup should remain in history") }) + + It("cleans deleted history and related rows only for the selected database", func() { + setupGpbackmanFilterDatabase() + DeferCleanup(teardownGpbackmanFilterDatabase) + + primaryOutput := gpbackup(gpbackupPath, backupHelperPath, + "--backup-dir", backupDir) + primaryTimestamp := getBackupTimestamp(string(primaryOutput)) + filterOutput := gpbackupForDatabase(gpbackmanFilterDatabase, gpbackupPath, backupHelperPath, + "--backup-dir", backupDir, + "--include-table", "public.e2e_data") + filterTimestamp := getBackupTimestamp(string(filterOutput)) + + gpbackman( + "backup-delete", + "--history-db", historyDB, + "--timestamp", primaryTimestamp, + ) + gpbackman( + "backup-delete", + "--history-db", historyDB, + "--timestamp", filterTimestamp, + ) + Expect(queryHistoryDB(historyDB, + fmt.Sprintf("SELECT count(*) FROM include_relations WHERE timestamp = '%s'", filterTimestamp))).To(Equal("1")) + + gpbackman( + "history-clean", + "--history-db", historyDB, + "--before-timestamp", "99991231235959", + "--database", gpbackmanFilterDatabase, + ) + + Expect(queryHistoryDB(historyDB, + fmt.Sprintf("SELECT count(*) FROM backups WHERE timestamp = '%s'", primaryTimestamp))).To(Equal("1")) + Expect(queryHistoryDB(historyDB, + fmt.Sprintf("SELECT count(*) FROM backups WHERE timestamp = '%s'", filterTimestamp))).To(Equal("0")) + Expect(queryHistoryDB(historyDB, + fmt.Sprintf("SELECT count(*) FROM include_relations WHERE timestamp = '%s'", filterTimestamp))).To(Equal("0")) + }) }) // ------------------------------------------------------------------ // diff --git a/gpbackman/COMMANDS.md b/gpbackman/COMMANDS.md index 715536b9..7f827f4f 100644 --- a/gpbackman/COMMANDS.md +++ b/gpbackman/COMMANDS.md @@ -65,6 +65,11 @@ To delete backup sets older than the given number of days, use the --older-than- To delete backup sets newer than the given timestamp, use the --after-timestamp option. Only --older-than-days, --before-timestamp or --after-timestamp option must be specified. +Use --database to clean backup sets only for the specified database. Without --database, +cleanup includes backup sets for all databases in the history database. +Database names are matched exactly and case-sensitively against backup history. +For database names that require quoting, include the double quotes in the --database value. + By default, the existence of dependent backups is checked and deletion process is not performed, unless the --cascade option is passed in. @@ -99,6 +104,7 @@ Flags: --backup-dir string the full path to backup directory for local backups --before-timestamp string delete backup sets older than the given timestamp --cascade delete all dependent backups + --database string delete backup sets only for the specified database -h, --help help for backup-clean --history-sync-standby-timeout int shared rsync and remote install timeout in seconds; must be an integer between 1 and 86400 (default 300) --no-history-sync-standby skip automatic gpbackup_history.db sync to standby coordinator after this command @@ -124,6 +130,20 @@ Delete backups older than a timestamp: --cascade ``` +Delete local backups only for database `analytics`: +```bash +./gpbackman backup-clean \ + --before-timestamp 20240701100000 \ + --database analytics +``` + +For database `Sales DB`, include double quotes in the flag value: +```bash +./gpbackman backup-clean \ + --before-timestamp 20240701100000 \ + --database '"Sales DB"' +``` + Delete backups older than a number of days with multiple parallel processes: ```bash ./gpbackman backup-clean \ @@ -251,6 +271,12 @@ To display all backups, use --deleted and --failed options together. To display backups of a specific type, use the --type option. +Without the --database option, backups for all databases are displayed. +To display backups only for a specific database, use the --database option. +Database names are matched exactly and case-sensitively against backup history. +The --database value is used without transformation. For database names that require quoting, +include the double quotes in the flag value, for example: --database '"Sales DB"'. + To display backups that include the specified table, use the --table option. The formatting rules for . match those of the --include-table option in gpbackup. @@ -273,7 +299,7 @@ To display a backup chain for a specific backup, use the --timestamp option. In this mode, the backup with the specified timestamp and all of its dependent backups will be displayed. The deleted and failed backups are always included in this mode. To display object filtering details in this mode, use the --detail option. -When --timestamp is set, the following options cannot be used: --type, --table, --schema, --exclude, --failed, --deleted. +When --timestamp is set, the following options cannot be used: --database, --type, --table, --schema, --exclude, --failed, --deleted. To display the "object filtering details" column for all backups without using --timestamp, use the --detail option. @@ -285,6 +311,7 @@ Usage: gpbackman backup-info [flags] Flags: + --database string show backups only for the specified database (exact, case-sensitive match) --deleted show deleted backups --detail show object filtering details --exclude show backups that exclude the specific table (format .
) or schema @@ -378,6 +405,19 @@ Display info for active full backups from `gpbackup_history.db`: 20230523101115 | Tue May 23 2023 10:11:15 | Success | demo | full | include-schema | gpbackup_s3_plugin | 01:01:00 | ``` +Display active backups only for database `analytics`: +```bash +./gpbackman backup-info \ + --database analytics +``` + +Display active full backups only for database `analytics`: +```bash +./gpbackman backup-info \ + --database analytics \ + --type full +``` + Find all backups, including deleted ones, containing the `test1` schema. ```bash ./gpbackman backup-info \ @@ -477,6 +517,11 @@ To delete information about backups older than the given timestamp, use the --be To delete information about backups older than the given number of days, use the --older-than-day option. Only --older-than-days or --before-timestamp option must be specified, not both. +Use --database to clean history only for the specified database. Without --database, +cleanup includes deleted backup history for all databases in the history database. +Database names are matched exactly and case-sensitively against backup history. +For database names that require quoting, include the double quotes in the --database value. + The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY instead, pass the --auto-load-history-db flag. @@ -486,6 +531,7 @@ Usage: Flags: --before-timestamp string delete information about backups older than the given timestamp + --database string delete backup history only for the specified database -h, --help help for history-clean --history-sync-standby-timeout int shared rsync and remote install timeout in seconds; must be an integer between 1 and 86400 (default 300) --no-history-sync-standby skip automatic gpbackup_history.db sync to standby coordinator after this command @@ -514,6 +560,20 @@ Delete information about deleted backups from history database older than timest --before-timestamp 20240101100000 ``` +Delete deleted backup history only for database `analytics`: +```bash +./gpbackman history-clean \ + --before-timestamp 20240101100000 \ + --database analytics +``` + +For database `Sales DB`, include double quotes in the flag value: +```bash +./gpbackman history-clean \ + --before-timestamp 20240101100000 \ + --database '"Sales DB"' +``` + # Sync the history database to the standby coordinator (`history-sync`) Available options for `history-sync` command and their description: diff --git a/gpbackman/cmd/backup_clean.go b/gpbackman/cmd/backup_clean.go index 5435c2c8..7222a555 100644 --- a/gpbackman/cmd/backup_clean.go +++ b/gpbackman/cmd/backup_clean.go @@ -42,6 +42,7 @@ var ( backupCleanParallelProcesses int backupCleanCascade bool backupCleanNoHistorySyncStandby bool + backupCleanDatabase string ) var backupCleanCmd = &cobra.Command{ @@ -54,6 +55,11 @@ To delete backup sets older than the given number of days, use the --older-than- To delete backup sets newer than the given timestamp, use the --after-timestamp option. Only --older-than-days, --before-timestamp or --after-timestamp option must be specified. +Use --database to clean backups only for the specified database. Without --database, +cleanup includes backups for all databases in the history database. +Database names are matched exactly and case-sensitively against backup history. +For database names that require quoting, include the double quotes in the --database value. + By default, the existence of dependent backups is checked and deletion process is not performed, unless the --cascade option is passed in. @@ -89,6 +95,12 @@ If the --history-db option is not specified, the history database is looked for func init() { rootCmd.AddCommand(backupCleanCmd) + backupCleanCmd.Flags().StringVar( + &backupCleanDatabase, + databaseFlagName, + "", + "delete backup sets only for the specified database", + ) backupCleanCmd.PersistentFlags().StringVar( &backupCleanPluginConfigFile, pluginConfigFileFlagName, @@ -149,6 +161,10 @@ func init() { // These flag checks are applied only for backup-clean command. func doCleanBackupFlagValidation(flags *pflag.FlagSet) { var err error + if flags.Changed(databaseFlagName) && backupCleanDatabase == "" { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupCleanDatabase, databaseFlagName, textmsg.ErrorEmptyDatabase())) + execOSExit(exitErrorCode) + } // If before-timestamp flag is specified and have correct values. if flags.Changed(beforeTimestampFlagName) { err = gpbckpconfig.CheckTimestamp(backupCleanBeforeTimestamp) @@ -232,12 +248,12 @@ func cleanBackup() error { gplog.Error("%s", textmsg.ErrorTextUnableReadPluginConfigFile(err)) return err } - err = backupCleanDBPlugin(backupCleanCascade, beforeTimestamp, afterTimestamp, backupCleanPluginConfigFile, pluginConfig, hDB) + err = backupCleanDBPlugin(backupCleanCascade, beforeTimestamp, afterTimestamp, backupCleanDatabase, backupCleanPluginConfigFile, pluginConfig, hDB) if err != nil { return err } } else { - err := backupCleanDBLocal(backupCleanCascade, beforeTimestamp, afterTimestamp, backupCleanBackupDir, backupCleanParallelProcesses, hDB) + err := backupCleanDBLocal(backupCleanCascade, beforeTimestamp, afterTimestamp, backupCleanDatabase, backupCleanBackupDir, backupCleanParallelProcesses, hDB) if err != nil { return err } @@ -245,8 +261,8 @@ func cleanBackup() error { return nil } -func backupCleanDBPlugin(deleteCascade bool, cutOffTimestamp, cutOffAfterTimestamp, pluginConfigPath string, pluginConfig *utils.PluginConfig, hDB *sql.DB) error { - backupList, err := fetchBackupNamesForDeletion(cutOffTimestamp, cutOffAfterTimestamp, hDB) +func backupCleanDBPlugin(deleteCascade bool, cutOffTimestamp, cutOffAfterTimestamp, databaseName, pluginConfigPath string, pluginConfig *utils.PluginConfig, hDB *sql.DB) error { + backupList, err := fetchBackupNamesForDeletion(cutOffTimestamp, cutOffAfterTimestamp, databaseName, hDB) if err != nil { gplog.Error("%s", textmsg.ErrorTextUnableReadHistoryDB(err)) return err @@ -266,8 +282,8 @@ func backupCleanDBPlugin(deleteCascade bool, cutOffTimestamp, cutOffAfterTimesta return nil } -func backupCleanDBLocal(deleteCascade bool, cutOffTimestamp, cutOffAfterTimestamp, backupDir string, maxParallelProcesses int, hDB *sql.DB) error { - backupList, err := fetchBackupNamesForDeletion(cutOffTimestamp, cutOffAfterTimestamp, hDB) +func backupCleanDBLocal(deleteCascade bool, cutOffTimestamp, cutOffAfterTimestamp, databaseName, backupDir string, maxParallelProcesses int, hDB *sql.DB) error { + backupList, err := fetchBackupNamesForDeletion(cutOffTimestamp, cutOffAfterTimestamp, databaseName, hDB) if err != nil { gplog.Error("%s", textmsg.ErrorTextUnableReadHistoryDB(err)) return err @@ -285,17 +301,17 @@ func backupCleanDBLocal(deleteCascade bool, cutOffTimestamp, cutOffAfterTimestam } // Get the list of backup names for deletion. -func fetchBackupNamesForDeletion(cutOffTimestamp, cutOffAfterTimestamp string, hDB *sql.DB) ([]string, error) { +func fetchBackupNamesForDeletion(cutOffTimestamp, cutOffAfterTimestamp, databaseName string, hDB *sql.DB) ([]string, error) { var backupList []string var err error if cutOffTimestamp != "" { - backupList, err = gpbckpconfig.GetBackupNamesBeforeTimestamp(cutOffTimestamp, hDB) + backupList, err = gpbckpconfig.GetBackupNamesBeforeTimestamp(cutOffTimestamp, databaseName, hDB) if err != nil { return nil, err } } if cutOffAfterTimestamp != "" { - backupList, err = gpbckpconfig.GetBackupNamesAfterTimestamp(cutOffAfterTimestamp, hDB) + backupList, err = gpbckpconfig.GetBackupNamesAfterTimestamp(cutOffAfterTimestamp, databaseName, hDB) if err != nil { return nil, err } diff --git a/gpbackman/cmd/backup_clean_test.go b/gpbackman/cmd/backup_clean_test.go new file mode 100644 index 00000000..ef7f4d9f --- /dev/null +++ b/gpbackman/cmd/backup_clean_test.go @@ -0,0 +1,132 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package cmd + +import ( + "database/sql" + "path/filepath" + + "github.com/spf13/pflag" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("backup-clean database filter", func() { + It("registers a command-local database flag and documents its behavior", func() { + flag := backupCleanCmd.Flags().Lookup(databaseFlagName) + Expect(flag).NotTo(BeNil()) + Expect(flag.DefValue).To(Equal("")) + Expect(flag.Usage).To(ContainSubstring("specified database")) + Expect(backupCleanCmd.Long).To(ContainSubstring("Without --database")) + Expect(backupCleanCmd.Long).To(ContainSubstring("case-sensitively")) + }) + + It("requires a value when the database flag is supplied", func() { + rootCmd.SetArgs([]string{"backup-clean", "--" + databaseFlagName}) + DeferCleanup(func() { rootCmd.SetArgs(nil) }) + + err := rootCmd.Execute() + Expect(err).To(MatchError(ContainSubstring("flag needs an argument"))) + }) + + DescribeTable("validates explicit empty database values", + func(database string, setDatabase, wantExit bool) { + oldDatabase := backupCleanDatabase + oldCleanBeforeTimestamp := backupCleanBeforeTimestamp + oldBeforeTimestamp := beforeTimestamp + oldAfterTimestamp := afterTimestamp + oldExecOSExit := execOSExit + DeferCleanup(func() { + backupCleanDatabase = oldDatabase + backupCleanBeforeTimestamp = oldCleanBeforeTimestamp + beforeTimestamp = oldBeforeTimestamp + afterTimestamp = oldAfterTimestamp + execOSExit = oldExecOSExit + }) + + backupCleanDatabase = database + backupCleanBeforeTimestamp = "20240101120000" + beforeTimestamp = "" + afterTimestamp = "" + flags := pflag.NewFlagSet("test", pflag.ContinueOnError) + flags.String(beforeTimestampFlagName, "", "") + flags.String(databaseFlagName, "", "") + Expect(flags.Set(beforeTimestampFlagName, backupCleanBeforeTimestamp)).To(Succeed()) + if setDatabase { + Expect(flags.Set(databaseFlagName, database)).To(Succeed()) + } + + exited := false + execOSExit = func(code int) { + Expect(code).To(Equal(exitErrorCode)) + exited = true + } + doCleanBackupFlagValidation(flags) + Expect(exited).To(Equal(wantExit)) + }, + Entry("when absent", "", false, false), + Entry("when non-empty", `"Customer's DB"`, true, false), + Entry("when explicitly empty", "", true, true), + ) + + Describe("backup selection", func() { + var historyDB *sql.DB + + BeforeEach(func() { + var err error + historyDB, err = sql.Open("sqlite3", "file:"+filepath.Join(GinkgoT().TempDir(), "history.db")+"?mode=rwc") + Expect(err).NotTo(HaveOccurred()) + _, err = historyDB.Exec(`CREATE TABLE backups (timestamp TEXT, database_name TEXT, status TEXT, date_deleted TEXT)`) + Expect(err).NotTo(HaveOccurred()) + for _, backup := range [][]string{ + {"20240101090000", "demo", "Success", ""}, + {"20240101100000", `"Customer's DB"`, "Success", ""}, + {"20240101110000", `"customer's db"`, "Success", ""}, + {"20240101130000", `"Customer's DB"`, "Success", ""}, + } { + _, err = historyDB.Exec(`INSERT INTO backups (timestamp, database_name, status, date_deleted) VALUES (?, ?, ?, ?)`, backup[0], backup[1], backup[2], backup[3]) + Expect(err).NotTo(HaveOccurred()) + } + }) + + AfterEach(func() { + Expect(historyDB.Close()).To(Succeed()) + }) + + DescribeTable("filters before and after timestamp selections exactly", + func(before, after, database string, expected []string) { + actual, err := fetchBackupNamesForDeletion(before, after, database, historyDB) + Expect(err).NotTo(HaveOccurred()) + Expect(actual).To(Equal(expected)) + }, + Entry("before timestamp exact database", "20240101120000", "", `"Customer's DB"`, []string{"20240101100000"}), + Entry("after timestamp exact database", "", "20240101120000", `"Customer's DB"`, []string{"20240101130000"}), + Entry("quoted case mismatch", "20240101120000", "", `"customer's db"`, []string{"20240101110000"}), + Entry("unknown database", "20240101120000", "", "unknown", nil), + Entry("without filter", "20240101120000", "", "", []string{"20240101110000", "20240101100000", "20240101090000"}), + ) + + It("does not invoke local or plugin cleanup for an unknown database", func() { + Expect(backupCleanDBLocal(false, "20240101120000", "", "unknown", "", 1, historyDB)).To(Succeed()) + Expect(backupCleanDBPlugin(false, "20240101120000", "", "unknown", "", nil, historyDB)).To(Succeed()) + }) + }) +}) diff --git a/gpbackman/cmd/backup_info.go b/gpbackman/cmd/backup_info.go index 96674b48..073a35eb 100644 --- a/gpbackman/cmd/backup_info.go +++ b/gpbackman/cmd/backup_info.go @@ -41,6 +41,7 @@ var ( backupInfoTableNameFilter string backupInfoSchemaNameFilter string backupInfoExcludeFilter bool + backupInfoDatabase string backupInfoTimestamp string backupInfoShowDetails bool ) @@ -53,6 +54,7 @@ type BackupInfoOptions struct { TableNameFilter string SchemaNameFilter string ExcludeFilter bool + DatabaseFilter string Timestamp string ShowDetails bool } @@ -70,6 +72,12 @@ To display all backups, use --deleted and --failed options together. To display backups of a specific type, use the --type option. +Without the --database option, backups for all databases are displayed. +To display backups only for a specific database, use the --database option. +Database names are matched exactly and case-sensitively against backup history. +The --database value is used without transformation. For database names that require quoting, +include the double quotes in the flag value, for example: --database '"Sales DB"'. + To display backups that include the specified table, use the --table option. The formatting rules for .
match those of the --include-table option in gpbackup. @@ -91,8 +99,8 @@ The details are presented as follows, depending on the active filtering type: To display a backup chain for a specific backup, use the --timestamp option. In this mode, the backup with the specified timestamp and all of its dependent backups will be displayed. The deleted and failed backups are always included in this mode. -To display object filtering details in this mode, use the --detail option. -When --timestamp is set, the following options cannot be used: --type, --table, --schema, --exclude, --failed, --deleted. +The --detail option can be used with --timestamp to display object filtering details in this mode. +When --timestamp is set, the following options cannot be used: --database, --type, --table, --schema, --exclude, --failed, --deleted. To display the "object filtering details" column for all backups without using --timestamp, use the --detail option. @@ -151,6 +159,12 @@ func init() { false, "show backups that exclude the specific table (format .
) or schema", ) + backupInfoCmd.Flags().StringVar( + &backupInfoDatabase, + databaseFlagName, + "", + "show backups only for the specified database (exact, case-sensitive match)", + ) backupInfoCmd.Flags().BoolVar( &backupInfoShowDetails, detailFlagName, @@ -162,17 +176,21 @@ func init() { // These flag checks are applied only for backup-info commands. func doBackupInfoFlagValidation(flags *pflag.FlagSet) { var err error + if flags.Changed(databaseFlagName) && backupInfoDatabase == "" { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupInfoDatabase, databaseFlagName, textmsg.ErrorEmptyDatabase())) + execOSExit(exitErrorCode) + } if flags.Changed(timestampFlagName) { err = gpbckpconfig.CheckTimestamp(backupInfoTimestamp) if err != nil { gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(backupInfoTimestamp, timestampFlagName, err)) execOSExit(exitErrorCode) } - // --timestamp is not compatible with --type, --table, --schema, --exclude, --failed, --deleted + // --timestamp is not compatible with --database, --type, --table, --schema, --exclude, --failed, --deleted err = checkCompatibleFlags(flags, timestampFlagName, - typeFlagName, tableFlagName, schemaFlagName, excludeFlagName, failedFlagName, deletedFlagName) + databaseFlagName, typeFlagName, tableFlagName, schemaFlagName, excludeFlagName, failedFlagName, deletedFlagName) if err != nil { - gplog.Error("%s", textmsg.ErrorTextUnableCompatibleFlags(err, timestampFlagName, typeFlagName, tableFlagName, schemaFlagName, excludeFlagName, failedFlagName, deletedFlagName)) + gplog.Error("%s", textmsg.ErrorTextUnableCompatibleFlags(err, timestampFlagName, databaseFlagName, typeFlagName, tableFlagName, schemaFlagName, excludeFlagName, failedFlagName, deletedFlagName)) execOSExit(exitErrorCode) } } @@ -221,6 +239,7 @@ func backupInfo() error { TableNameFilter: backupInfoTableNameFilter, SchemaNameFilter: backupInfoSchemaNameFilter, ExcludeFilter: backupInfoExcludeFilter, + DatabaseFilter: backupInfoDatabase, Timestamp: backupInfoTimestamp, ShowDetails: backupInfoShowDetails, } @@ -259,7 +278,7 @@ func backupInfoDB(opts BackupInfoOptions, hDB *sql.DB, t *tablewriter.Table) err gplog.Error("%s", textmsg.ErrorTextUnableGetBackupInfo(backupName, err)) return err } - addBackupToTable(opts.BackupTypeFilter, opts.TableNameFilter, opts.SchemaNameFilter, opts.ExcludeFilter, opts.ShowDetails, backupData, t) + addBackupToTable(opts.BackupTypeFilter, opts.TableNameFilter, opts.SchemaNameFilter, opts.DatabaseFilter, opts.ExcludeFilter, opts.ShowDetails, backupData, t) } return nil } @@ -270,7 +289,7 @@ func backupInfoDB(opts BackupInfoOptions, hDB *sql.DB, t *tablewriter.Table) err gplog.Error("%s", textmsg.ErrorTextUnableGetBackupInfo(opts.Timestamp, err)) return err } - addBackupToTable("", "", "", false, opts.ShowDetails, baseBackupData, t) + addBackupToTable("", "", "", "", false, opts.ShowDetails, baseBackupData, t) backupDependenciesList, err := gpbckpconfig.GetBackupDependencies(opts.Timestamp, hDB) if err != nil { gplog.Error("%s", textmsg.ErrorTextUnableReadHistoryDB(err)) @@ -282,7 +301,7 @@ func backupInfoDB(opts BackupInfoOptions, hDB *sql.DB, t *tablewriter.Table) err gplog.Error("%s", textmsg.ErrorTextUnableGetBackupInfo(depTimestamp, err)) return err } - addBackupToTable("", "", "", false, opts.ShowDetails, backupData, t) + addBackupToTable("", "", "", "", false, opts.ShowDetails, backupData, t) } return nil } @@ -311,7 +330,11 @@ func initTable(t *tablewriter.Table, includeDetails bool) { // If errors occur, they are logged, but they are not returned. // The main idea is to show the maximum available information and display all errors that occur. // But do not fall when errors occur. So, display anyway. -func addBackupToTable(backupTypeFilter, backupTableFilter, backupSchemaFilter string, backupExcludeFilter, includeDetails bool, backupData *history.BackupConfig, t *tablewriter.Table) { +func addBackupToTable(backupTypeFilter, backupTableFilter, backupSchemaFilter, backupDatabaseFilter string, backupExcludeFilter, includeDetails bool, backupData *history.BackupConfig, t *tablewriter.Table) { + if backupDatabaseFilter != "" && backupDatabaseFilter != backupData.DatabaseName { + return + } + var matchToObjectFilter bool backupDate, err := gpbckpconfig.GetBackupDate(backupData) if err != nil { diff --git a/gpbackman/cmd/backup_info_test.go b/gpbackman/cmd/backup_info_test.go new file mode 100644 index 00000000..a9405840 --- /dev/null +++ b/gpbackman/cmd/backup_info_test.go @@ -0,0 +1,292 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package cmd + +import ( + "database/sql" + "path/filepath" + + "github.com/apache/cloudberry-backup/history" + "github.com/olekukonko/tablewriter" + "github.com/spf13/pflag" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("backup-info database filter", func() { + It("registers and documents the command-local database flag", func() { + flag := backupInfoCmd.Flags().Lookup(databaseFlagName) + Expect(flag).NotTo(BeNil()) + Expect(flag.DefValue).To(Equal("")) + Expect(flag.Usage).To(ContainSubstring("specified database")) + Expect(backupInfoCmd.Long).To(ContainSubstring("Without the --database option, backups for all databases are displayed.")) + Expect(backupInfoCmd.Long).To(ContainSubstring("Database names are matched exactly and case-sensitively against backup history.")) + Expect(backupInfoCmd.Long).To(ContainSubstring("include the double quotes in the flag value")) + Expect(backupInfoCmd.Long).To(ContainSubstring("The --detail option can be used with --timestamp")) + Expect(backupInfoCmd.Long).To(ContainSubstring("--database, --type, --table, --schema, --exclude, --failed, --deleted")) + Expect(backupInfoCmd.UsageString()).To(ContainSubstring("--" + databaseFlagName + " string")) + }) + + It("requires a value when the database flag is supplied", func() { + rootCmd.SetArgs([]string{"backup-info", "--" + databaseFlagName}) + DeferCleanup(func() { rootCmd.SetArgs(nil) }) + + Expect(rootCmd.Execute()).To(MatchError(ContainSubstring("flag needs an argument"))) + }) + + DescribeTable("validates explicit database values", + func(database string, setDatabase, wantExit bool) { + oldDatabase := backupInfoDatabase + oldTimestamp := backupInfoTimestamp + oldExecOSExit := execOSExit + DeferCleanup(func() { + backupInfoDatabase = oldDatabase + backupInfoTimestamp = oldTimestamp + execOSExit = oldExecOSExit + }) + + backupInfoDatabase = database + backupInfoTimestamp = "" + flags := pflag.NewFlagSet("backup-info", pflag.ContinueOnError) + flags.String(databaseFlagName, "", "") + if setDatabase { + Expect(flags.Set(databaseFlagName, database)).To(Succeed()) + } + + exited := false + execOSExit = func(code int) { + Expect(code).To(Equal(exitErrorCode)) + exited = true + } + doBackupInfoFlagValidation(flags) + Expect(exited).To(Equal(wantExit)) + }, + Entry("when absent", "", false, false), + Entry("when non-empty", `"Customer's DB"`, true, false), + Entry("when explicitly empty", "", true, true), + ) + + DescribeTable("preserves timestamp and detail compatibility", + func(setDatabase, setTimestamp, setDetail, wantExit bool) { + oldDatabase := backupInfoDatabase + oldTimestamp := backupInfoTimestamp + oldExecOSExit := execOSExit + DeferCleanup(func() { + backupInfoDatabase = oldDatabase + backupInfoTimestamp = oldTimestamp + execOSExit = oldExecOSExit + }) + + backupInfoDatabase = `"Customer's DB"` + backupInfoTimestamp = "20240101120000" + flags := pflag.NewFlagSet("backup-info", pflag.ContinueOnError) + flags.String(databaseFlagName, "", "") + flags.String(timestampFlagName, "", "") + flags.Bool(detailFlagName, false, "") + if setDatabase { + Expect(flags.Set(databaseFlagName, backupInfoDatabase)).To(Succeed()) + } + if setTimestamp { + Expect(flags.Set(timestampFlagName, backupInfoTimestamp)).To(Succeed()) + } + if setDetail { + Expect(flags.Set(detailFlagName, "true")).To(Succeed()) + } + + exited := false + execOSExit = func(code int) { + Expect(code).To(Equal(exitErrorCode)) + exited = true + } + doBackupInfoFlagValidation(flags) + Expect(exited).To(Equal(wantExit)) + }, + Entry("database with detail", true, false, true, false), + Entry("timestamp with detail", false, true, true, false), + Entry("database with timestamp", true, true, false, true), + Entry("database with timestamp and detail", true, true, true, true), + ) + + DescribeTable("matches database names exactly before other filters", + func(databaseFilter, backupDatabase string, wantRows int) { + t := tablewriter.NewWriter(GinkgoWriter) + backupData := backupInfoTestConfig("20240101000000", backupDatabase) + addBackupToTable("", "", "", databaseFilter, false, false, &backupData, t) + Expect(t.NumLines()).To(Equal(wantRows)) + }, + Entry("without a filter", "", "demo", 1), + Entry("with an exact match", "demo", "demo", 1), + Entry("with a case mismatch", "Demo", "demo", 0), + Entry("with a quoted name", `"Customer's DB"`, `"Customer's DB"`, 1), + Entry("with an unknown database", "unknown", "demo", 0), + ) + + It("displays matching backups when their derived fields are invalid", func() { + t := tablewriter.NewWriter(GinkgoWriter) + backupData := backupInfoTestConfig("invalid", "demo") + backupData.EndTime = "also-invalid" + backupData.Incremental = true + backupData.DataOnly = true + backupData.IncludeSchemaFiltered = true + backupData.IncludeTableFiltered = true + backupData.DateDeleted = "invalid" + + addBackupToTable("", "", "", "demo", false, false, &backupData, t) + Expect(t.NumLines()).To(Equal(1)) + }) + + It("composes database filtering with type, table, schema, and detail filters", func() { + matchingTable := backupInfoTestConfig("20240101000000", "demo") + matchingTable.IncludeTableFiltered = true + matchingTable.IncludeRelations = []string{"public.orders"} + matchingSchema := backupInfoTestConfig("20240101000001", "demo") + matchingSchema.IncludeSchemaFiltered = true + matchingSchema.IncludeSchemas = []string{"public"} + wrongType := backupInfoTestConfig("20240101000002", "demo") + wrongType.Incremental = true + wrongObject := backupInfoTestConfig("20240101000003", "demo") + wrongObject.IncludeTableFiltered = true + wrongObject.IncludeRelations = []string{"public.customers"} + otherDatabase := backupInfoTestConfig("20240101000004", "other") + otherDatabase.IncludeTableFiltered = true + otherDatabase.IncludeRelations = []string{"public.orders"} + configs := []*history.BackupConfig{&matchingTable, &matchingSchema, &wrongType, &wrongObject, &otherDatabase} + + for _, test := range []struct { + name, typeFilter, tableFilter, schemaFilter string + includeDetail bool + wantRows int + }{ + {"type", "full", "", "", false, 3}, + {"table", "", "public.orders", "", false, 1}, + {"schema", "", "", "public", false, 1}, + {"database and type", "full", "", "", false, 3}, + {"database and table", "", "public.orders", "", false, 1}, + {"database and schema", "", "", "public", false, 1}, + {"database and detail", "full", "", "", true, 3}, + } { + t := tablewriter.NewWriter(GinkgoWriter) + for _, config := range configs { + addBackupToTable(test.typeFilter, test.tableFilter, test.schemaFilter, "demo", false, test.includeDetail, config, t) + } + Expect(t.NumLines()).To(Equal(test.wantRows), test.name) + } + }) + + It("filters database output and leaves timestamp chains unfiltered", func() { + baseTimestamp := "20240101000000" + historyDB := createBackupInfoTestDB( + backupInfoTestConfig(baseTimestamp, "demo"), + backupInfoTestConfig("20240101000001", "other"), + backupInfoTestConfigWithDependency("20240101000002", "demo", baseTimestamp), + backupInfoTestConfigWithDependency("20240101000003", `"Customer's DB"`, baseTimestamp), + ) + + filtered := tablewriter.NewWriter(GinkgoWriter) + Expect(backupInfoDB(BackupInfoOptions{DatabaseFilter: "demo"}, historyDB, filtered)).To(Succeed()) + Expect(filtered.NumLines()).To(Equal(2)) + + unknown := tablewriter.NewWriter(GinkgoWriter) + Expect(backupInfoDB(BackupInfoOptions{DatabaseFilter: "unknown"}, historyDB, unknown)).To(Succeed()) + Expect(unknown.NumLines()).To(Equal(0)) + + chain := tablewriter.NewWriter(GinkgoWriter) + Expect(backupInfoDB(BackupInfoOptions{Timestamp: baseTimestamp, ShowDetails: true}, historyDB, chain)).To(Succeed()) + Expect(chain.NumLines()).To(Equal(3)) + }) + + It("composes database filtering with exclude, deleted, and failed filters", func() { + configs := make([]history.BackupConfig, 0, 6) + for _, database := range []struct { + name string + timestamps [3]string + }{ + {"demo", [3]string{"20240101000000", "20240101000001", "20240101000002"}}, + {"other", [3]string{"20240101000003", "20240101000004", "20240101000005"}}, + } { + active := backupInfoTestConfig(database.timestamps[0], database.name) + active.ExcludeTableFiltered = true + active.ExcludeRelations = []string{"public.orders"} + configs = append(configs, active) + + deleted := backupInfoTestConfig(database.timestamps[1], database.name) + deleted.ExcludeTableFiltered = true + deleted.ExcludeRelations = []string{"public.orders"} + deleted.DateDeleted = "20240102000000" + configs = append(configs, deleted) + + failed := backupInfoTestConfig(database.timestamps[2], database.name) + failed.ExcludeTableFiltered = true + failed.ExcludeRelations = []string{"public.orders"} + failed.Status = history.BackupStatusFailed + configs = append(configs, failed) + } + historyDB := createBackupInfoTestDB(configs...) + + for _, test := range []struct { + name string + showDeleted bool + showFailed bool + wantRows int + }{ + {"exclude", false, false, 1}, + {"exclude and deleted", true, false, 2}, + {"exclude and failed", false, true, 2}, + {"exclude, deleted, and failed", true, true, 3}, + } { + t := tablewriter.NewWriter(GinkgoWriter) + err := backupInfoDB(BackupInfoOptions{ + ShowDeleted: test.showDeleted, + ShowFailed: test.showFailed, + TableNameFilter: "public.orders", + ExcludeFilter: true, + DatabaseFilter: "demo", + }, historyDB, t) + Expect(err).To(Succeed(), test.name) + Expect(t.NumLines()).To(Equal(test.wantRows), test.name) + } + }) +}) + +func backupInfoTestConfig(timestamp, databaseName string) history.BackupConfig { + return history.BackupConfig{ + Timestamp: timestamp, + EndTime: timestamp, + DatabaseName: databaseName, + Status: history.BackupStatusSucceed, + } +} + +func backupInfoTestConfigWithDependency(timestamp, databaseName, baseTimestamp string) history.BackupConfig { + config := backupInfoTestConfig(timestamp, databaseName) + config.RestorePlan = []history.RestorePlanEntry{{Timestamp: baseTimestamp}} + return config +} + +func createBackupInfoTestDB(configs ...history.BackupConfig) *sql.DB { + historyDB, err := history.InitializeHistoryDatabase(filepath.Join(GinkgoT().TempDir(), historyDBNameConst)) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { Expect(historyDB.Close()).To(Succeed()) }) + for i := range configs { + Expect(history.StoreBackupHistory(historyDB, &configs[i])).To(Succeed()) + } + return historyDB +} diff --git a/gpbackman/cmd/constants.go b/gpbackman/cmd/constants.go index 3d00a615..f068e9e2 100644 --- a/gpbackman/cmd/constants.go +++ b/gpbackman/cmd/constants.go @@ -55,6 +55,7 @@ const ( excludeFlagName = "exclude" backupDirFlagName = "backup-dir" parallelProcessesFlagName = "parallel-processes" + databaseFlagName = "database" ignoreErrorsFlagName = "ignore-errors" noHistorySyncStandbyFlagName = "no-history-sync-standby" historySyncStandbyTimeoutFlagName = "history-sync-standby-timeout" diff --git a/gpbackman/cmd/history_clean.go b/gpbackman/cmd/history_clean.go index b8012339..3ad318b4 100644 --- a/gpbackman/cmd/history_clean.go +++ b/gpbackman/cmd/history_clean.go @@ -35,6 +35,7 @@ var ( historyCleanBeforeTimestamp string historyCleanOlderThanDays uint historyCleanNoHistorySyncStandby bool + historyCleanDatabase string ) var historyCleanCmd = &cobra.Command{ @@ -49,6 +50,11 @@ To delete information about backups older than the given timestamp, use the --be To delete information about backups older than the given number of days, use the --older-than-day option. Only --older-than-days or --before-timestamp option must be specified, not both. +Use --database to clean history only for the specified database. Without --database, +cleanup includes deleted backup history for all databases in the history database. +Database names are matched exactly and case-sensitively against backup history. +For database names that require quoting, include the double quotes in the --database value. + The gpbackup_history.db file location can be set using the --history-db option. Can be specified only once. The full path to the file is required. If the --history-db option is not specified, the history database is looked for in the current directory. To resolve it from $COORDINATOR_DATA_DIRECTORY instead, pass the --auto-load-history-db flag.`, @@ -62,6 +68,12 @@ If the --history-db option is not specified, the history database is looked for func init() { rootCmd.AddCommand(historyCleanCmd) + historyCleanCmd.Flags().StringVar( + &historyCleanDatabase, + databaseFlagName, + "", + "delete backup history only for the specified database", + ) historyCleanCmd.PersistentFlags().UintVar( &historyCleanOlderThanDays, olderThanDaysFlagName, @@ -92,6 +104,10 @@ func init() { // These flag checks are applied only for history-clean command. func doCleanHistoryFlagValidation(flags *pflag.FlagSet) { var err error + if flags.Changed(databaseFlagName) && historyCleanDatabase == "" { + gplog.Error("%s", textmsg.ErrorTextUnableValidateFlag(historyCleanDatabase, databaseFlagName, textmsg.ErrorEmptyDatabase())) + execOSExit(exitErrorCode) + } // If before-timestamp are specified and have correct values. if flags.Changed(beforeTimestampFlagName) { err = gpbckpconfig.CheckTimestamp(historyCleanBeforeTimestamp) @@ -127,15 +143,15 @@ func cleanHistory() error { gplog.Error("%s", textmsg.ErrorTextUnableActionHistoryDB("close", closeErr)) } }() - err = historyCleanDB(beforeTimestamp, hDB) + err = historyCleanDB(beforeTimestamp, historyCleanDatabase, hDB) if err != nil { return err } return nil } -func historyCleanDB(cutOffTimestamp string, hDB *sql.DB) error { - backupList, err := gpbckpconfig.GetBackupNamesForCleanBeforeTimestamp(cutOffTimestamp, hDB) +func historyCleanDB(cutOffTimestamp, databaseName string, hDB *sql.DB) error { + backupList, err := gpbckpconfig.GetBackupNamesForCleanBeforeTimestamp(cutOffTimestamp, databaseName, hDB) if err != nil { gplog.Error("%s", textmsg.ErrorTextUnableReadHistoryDB(err)) return err diff --git a/gpbackman/cmd/history_clean_test.go b/gpbackman/cmd/history_clean_test.go new file mode 100644 index 00000000..9edda0fb --- /dev/null +++ b/gpbackman/cmd/history_clean_test.go @@ -0,0 +1,182 @@ +/* +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +*/ + +package cmd + +import ( + "database/sql" + "path/filepath" + + "github.com/apache/cloudberry-backup/history" + "github.com/spf13/pflag" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" +) + +var _ = Describe("history-clean database filter", func() { + It("registers a command-local database flag and documents its behavior", func() { + flag := historyCleanCmd.Flags().Lookup(databaseFlagName) + Expect(flag).NotTo(BeNil()) + Expect(flag.DefValue).To(Equal("")) + Expect(flag.Usage).To(ContainSubstring("specified database")) + Expect(historyCleanCmd.Long).To(ContainSubstring("Without --database")) + Expect(historyCleanCmd.Long).To(ContainSubstring("case-sensitively")) + }) + + It("requires a value when the database flag is supplied", func() { + rootCmd.SetArgs([]string{"history-clean", "--" + databaseFlagName}) + DeferCleanup(func() { rootCmd.SetArgs(nil) }) + + err := rootCmd.Execute() + Expect(err).To(MatchError(ContainSubstring("flag needs an argument"))) + }) + + DescribeTable("validates explicit empty database values", + func(database string, setDatabase, olderThan, wantExit bool) { + oldDatabase := historyCleanDatabase + oldCleanBeforeTimestamp := historyCleanBeforeTimestamp + oldCleanOlderThanDays := historyCleanOlderThanDays + oldBeforeTimestamp := beforeTimestamp + oldExecOSExit := execOSExit + DeferCleanup(func() { + historyCleanDatabase = oldDatabase + historyCleanBeforeTimestamp = oldCleanBeforeTimestamp + historyCleanOlderThanDays = oldCleanOlderThanDays + beforeTimestamp = oldBeforeTimestamp + execOSExit = oldExecOSExit + }) + + historyCleanDatabase = database + historyCleanBeforeTimestamp = "20240101120000" + historyCleanOlderThanDays = 1 + beforeTimestamp = "" + flags := pflag.NewFlagSet("test", pflag.ContinueOnError) + flags.String(beforeTimestampFlagName, "", "") + flags.Uint(olderThanDaysFlagName, 0, "") + flags.String(databaseFlagName, "", "") + if olderThan { + Expect(flags.Set(olderThanDaysFlagName, "1")).To(Succeed()) + } else { + Expect(flags.Set(beforeTimestampFlagName, historyCleanBeforeTimestamp)).To(Succeed()) + } + if setDatabase { + Expect(flags.Set(databaseFlagName, database)).To(Succeed()) + } + + exited := false + execOSExit = func(code int) { + Expect(code).To(Equal(exitErrorCode)) + exited = true + } + doCleanHistoryFlagValidation(flags) + Expect(exited).To(Equal(wantExit)) + if olderThan { + Expect(beforeTimestamp).NotTo(BeEmpty()) + } + }, + Entry("when absent", "", false, false, false), + Entry("when non-empty", `"Customer's DB"`, true, false, false), + Entry("when explicitly empty", "", true, false, true), + Entry("with older-than-days", "demo", true, true, false), + ) + + It("cleans only the selected database and its related history rows", func() { + historyDB := createHistoryCleanTestDB() + selectedDeleted := historyCleanTestConfig("20240101000000", `"Customer's DB"`, "20240102000000") + otherDeleted := historyCleanTestConfig("20240101010000", `"customer's db"`, "20240102000000") + selectedNew := historyCleanTestConfig("20240103000000", `"Customer's DB"`, "20240104000000") + selectedActive := historyCleanTestConfig("20240101020000", `"Customer's DB"`, "") + selectedFailed := historyCleanTestConfig("20240101030000", `"Customer's DB"`, "") + selectedFailed.Status = history.BackupStatusFailed + storeHistoryCleanConfigs(historyDB, selectedDeleted, otherDeleted, selectedNew, selectedActive, selectedFailed) + + Expect(historyCleanDB("20240102000000", `"Customer's DB"`, historyDB)).To(Succeed()) + + assertHistoryCleanTimestampExists(historyDB, selectedDeleted.Timestamp, false) + for _, backup := range []history.BackupConfig{otherDeleted, selectedNew, selectedActive, selectedFailed} { + assertHistoryCleanTimestampExists(historyDB, backup.Timestamp, true) + } + }) + + It("leaves unknown databases untouched and cleans all databases without a filter", func() { + historyDB := createHistoryCleanTestDB() + first := historyCleanTestConfig("20240101000000", "demo", "20240102000000") + second := historyCleanTestConfig("20240101010000", "other", "20240102000000") + storeHistoryCleanConfigs(historyDB, first, second) + + Expect(historyCleanDB("20240102000000", "unknown", historyDB)).To(Succeed()) + assertHistoryCleanTimestampExists(historyDB, first.Timestamp, true) + assertHistoryCleanTimestampExists(historyDB, second.Timestamp, true) + + Expect(historyCleanDB("20240102000000", "", historyDB)).To(Succeed()) + assertHistoryCleanTimestampExists(historyDB, first.Timestamp, false) + assertHistoryCleanTimestampExists(historyDB, second.Timestamp, false) + }) +}) + +var historyCleanAuxiliaryTables = []string{ + "backups", + "restore_plans", + "restore_plan_tables", + "exclude_relations", + "exclude_schemas", + "include_relations", + "include_schemas", +} + +func createHistoryCleanTestDB() *sql.DB { + historyDB, err := history.InitializeHistoryDatabase(filepath.Join(GinkgoT().TempDir(), "gpbackup_history.db")) + Expect(err).NotTo(HaveOccurred()) + _, err = historyDB.Exec("PRAGMA foreign_keys = OFF;") + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { Expect(historyDB.Close()).To(Succeed()) }) + return historyDB +} + +func historyCleanTestConfig(timestamp, databaseName, dateDeleted string) history.BackupConfig { + return history.BackupConfig{ + Timestamp: timestamp, + DatabaseName: databaseName, + DateDeleted: dateDeleted, + Status: history.BackupStatusSucceed, + ExcludeRelations: []string{"public.excluded_relation"}, + ExcludeSchemas: []string{"excluded_schema"}, + IncludeRelations: []string{"public.included_relation"}, + IncludeSchemas: []string{"included_schema"}, + RestorePlan: []history.RestorePlanEntry{{ + Timestamp: "20230101000000", + TableFQNs: []string{"public.restored_table"}, + }}, + } +} + +func storeHistoryCleanConfigs(historyDB *sql.DB, configs ...history.BackupConfig) { + for i := range configs { + Expect(history.StoreBackupHistory(historyDB, &configs[i])).To(Succeed()) + } +} + +func assertHistoryCleanTimestampExists(historyDB *sql.DB, timestamp string, want bool) { + for _, table := range historyCleanAuxiliaryTables { + var count int + Expect(historyDB.QueryRow("SELECT COUNT(*) FROM "+table+" WHERE timestamp = ?", timestamp).Scan(&count)).To(Succeed()) + Expect(count > 0).To(Equal(want), table) + } +} diff --git a/gpbackman/cmd/history_sync_test.go b/gpbackman/cmd/history_sync_test.go index acebdda3..9617ea6f 100644 --- a/gpbackman/cmd/history_sync_test.go +++ b/gpbackman/cmd/history_sync_test.go @@ -299,6 +299,26 @@ var _ = Describe("history sync command", func() { Expect(disabledValues).To(Equal([]bool{true, false, true})) }) + + It("keeps standby synchronization enabled for filtered cleanup commands", func() { + disabledValues := make([]bool, 0) + originalBackupCleanDatabase := backupCleanDatabase + originalHistoryCleanDatabase := historyCleanDatabase + runHistoryMutationWithStandbySync = func(work func() error, disabled bool) { + disabledValues = append(disabledValues, disabled) + } + backupCleanDatabase = `"Customer's DB"` + historyCleanDatabase = `"Customer's DB"` + DeferCleanup(func() { + backupCleanDatabase = originalBackupCleanDatabase + historyCleanDatabase = originalHistoryCleanDatabase + }) + + doCleanBackup() + doCleanHistory() + + Expect(disabledValues).To(Equal([]bool{false, false})) + }) }) }) diff --git a/gpbackman/cmd/wrappers_test.go b/gpbackman/cmd/wrappers_test.go index 12342ae5..f892cd08 100644 --- a/gpbackman/cmd/wrappers_test.go +++ b/gpbackman/cmd/wrappers_test.go @@ -33,6 +33,16 @@ import ( ) var _ = Describe("wrappers tests", func() { + Describe("database flag scope", func() { + It("registers the database filter only on supported commands", func() { + Expect(rootCmd.PersistentFlags().Lookup(databaseFlagName)).To(BeNil()) + for _, command := range rootCmd.Commands() { + want := command == backupCleanCmd || command == backupInfoCmd || command == historyCleanCmd + Expect(command.Flags().Lookup(databaseFlagName) != nil).To(Equal(want), command.Name()) + } + }) + }) + Describe("getHistoryDBPath", func() { // Save and restore env vars so these cases don't leak into the rest // of the suite when run with --randomize-all. diff --git a/gpbackman/gpbckpconfig/utils_db.go b/gpbackman/gpbckpconfig/utils_db.go index 48cc28d7..cd022feb 100644 --- a/gpbackman/gpbckpconfig/utils_db.go +++ b/gpbackman/gpbckpconfig/utils_db.go @@ -73,16 +73,16 @@ func GetBackupDependencies(backupName string, historyDB *sql.DB) ([]string, erro return execQueryFunc(getBackupDependenciesQuery(backupName), historyDB) } -func GetBackupNamesBeforeTimestamp(timestamp string, historyDB *sql.DB) ([]string, error) { - return execQueryFunc(getBackupNameBeforeTimestampQuery(timestamp), historyDB) +func GetBackupNamesBeforeTimestamp(timestamp, databaseName string, historyDB *sql.DB) ([]string, error) { + return execBackupNamesQuery(getBackupNameBeforeTimestampQuery(timestamp, databaseName), databaseName, historyDB) } -func GetBackupNamesAfterTimestamp(timestamp string, historyDB *sql.DB) ([]string, error) { - return execQueryFunc(getBackupNameAfterTimestampQuery(timestamp), historyDB) +func GetBackupNamesAfterTimestamp(timestamp, databaseName string, historyDB *sql.DB) ([]string, error) { + return execBackupNamesQuery(getBackupNameAfterTimestampQuery(timestamp, databaseName), databaseName, historyDB) } -func GetBackupNamesForCleanBeforeTimestamp(timestamp string, historyDB *sql.DB) ([]string, error) { - return execQueryFunc(getBackupNameForCleanBeforeTimestampQuery(timestamp), historyDB) +func GetBackupNamesForCleanBeforeTimestamp(timestamp, databaseName string, historyDB *sql.DB) ([]string, error) { + return execBackupNamesQuery(getBackupNameForCleanBeforeTimestampQuery(timestamp, databaseName), databaseName, historyDB) } func getBackupNameQuery(showD, showF bool) string { @@ -111,36 +111,50 @@ ORDER BY timestamp DESC; `, backupName, backupName) } -func getBackupNameBeforeTimestampQuery(timestamp string) string { - return fmt.Sprintf(` +func getBackupNameBeforeTimestampQuery(timestamp, databaseName string) string { + query := fmt.Sprintf(` SELECT timestamp FROM backups WHERE timestamp < '%s' AND status != '%s' AND date_deleted IN ('', '%s', '%s') -ORDER BY timestamp DESC; `, timestamp, history.BackupStatusInProgress, DateDeletedPluginFailed, DateDeletedLocalFailed) + return addDatabaseNamePredicate(query, databaseName) + "ORDER BY timestamp DESC;\n" } -func getBackupNameAfterTimestampQuery(timestamp string) string { - return fmt.Sprintf(` +func getBackupNameAfterTimestampQuery(timestamp, databaseName string) string { + query := fmt.Sprintf(` SELECT timestamp FROM backups WHERE timestamp > '%s' AND status != '%s' AND date_deleted IN ('', '%s', '%s') -ORDER BY timestamp DESC; `, timestamp, history.BackupStatusInProgress, DateDeletedPluginFailed, DateDeletedLocalFailed) + return addDatabaseNamePredicate(query, databaseName) + "ORDER BY timestamp DESC;\n" } -func getBackupNameForCleanBeforeTimestampQuery(timestamp string) string { - return fmt.Sprintf(` +func getBackupNameForCleanBeforeTimestampQuery(timestamp, databaseName string) string { + query := fmt.Sprintf(` SELECT timestamp FROM backups WHERE timestamp < '%s' AND date_deleted NOT IN ('', '%s', '%s', '%s') -ORDER BY timestamp DESC; `, timestamp, DateDeletedPluginFailed, DateDeletedLocalFailed, DateDeletedInProgress) + return addDatabaseNamePredicate(query, databaseName) + "ORDER BY timestamp DESC;\n" +} + +func addDatabaseNamePredicate(query, databaseName string) string { + if databaseName == "" { + return query + } + return query + "\tAND database_name = ?\n" +} + +func execBackupNamesQuery(query, databaseName string, historyDB *sql.DB) ([]string, error) { + if databaseName == "" { + return execQueryFunc(query, historyDB) + } + return execQueryFunc(query, historyDB, databaseName) } // UpdateDeleteStatus updates the date_deleted column in the history database. @@ -197,8 +211,8 @@ func updateDeleteStatusQuery(timestamp, status string) string { return fmt.Sprintf(`UPDATE backups SET date_deleted = '%s' WHERE timestamp = '%s';`, status, timestamp) } -func execQueryFunc(query string, historyDB *sql.DB) ([]string, error) { - sqlRow, err := historyDB.Query(query) +func execQueryFunc(query string, historyDB *sql.DB, args ...any) ([]string, error) { + sqlRow, err := historyDB.Query(query, args...) if err != nil { return nil, err } diff --git a/gpbackman/gpbckpconfig/utils_db_test.go b/gpbackman/gpbckpconfig/utils_db_test.go index a89e93e9..8582a842 100644 --- a/gpbackman/gpbckpconfig/utils_db_test.go +++ b/gpbackman/gpbckpconfig/utils_db_test.go @@ -128,7 +128,7 @@ WHERE timestamp < '20240101120000' AND date_deleted IN ('', 'Plugin Backup Delete Failed', 'Local Delete Failed') ORDER BY timestamp DESC; `, history.BackupStatusInProgress) - Expect(getBackupNameBeforeTimestampQuery("20240101120000")).To(Equal(want)) + Expect(getBackupNameBeforeTimestampQuery("20240101120000", "")).To(Equal(want)) }) }) @@ -142,7 +142,7 @@ WHERE timestamp > '20240101120000' AND date_deleted IN ('', 'Plugin Backup Delete Failed', 'Local Delete Failed') ORDER BY timestamp DESC; `, history.BackupStatusInProgress) - Expect(getBackupNameAfterTimestampQuery("20240101120000")).To(Equal(want)) + Expect(getBackupNameAfterTimestampQuery("20240101120000", "")).To(Equal(want)) }) }) @@ -155,10 +155,82 @@ WHERE timestamp < '20240101120000' AND date_deleted NOT IN ('', 'Plugin Backup Delete Failed', 'Local Delete Failed', 'In progress') ORDER BY timestamp DESC; ` - Expect(getBackupNameForCleanBeforeTimestampQuery("20240101120000")).To(Equal(want)) + Expect(getBackupNameForCleanBeforeTimestampQuery("20240101120000", "")).To(Equal(want)) }) }) + Describe("database-filtered backup name queries", func() { + var historyDB *sql.DB + + BeforeEach(func() { + var err error + historyDB, err = sql.Open("sqlite3", "file:"+filepath.Join(GinkgoT().TempDir(), "history.db")+"?mode=rwc") + Expect(err).NotTo(HaveOccurred()) + _, err = historyDB.Exec(`CREATE TABLE backups (timestamp TEXT, database_name TEXT, status TEXT, date_deleted TEXT)`) + Expect(err).NotTo(HaveOccurred()) + for _, backup := range [][]string{ + {"20240101110000", "customer", "Success", ""}, + {"20240101100000", "Customer", "Success", ""}, + {"20240101090000", `"quoted db"`, "Success", ""}, + {"20240101080000", "customer's db", "Success", ""}, + {"20240102110000", "customer", "Success", ""}, + {"20240102100000", "Customer", "Success", ""}, + {"20240102090000", `"quoted db"`, "Success", ""}, + {"20240102080000", "customer's db", "Success", ""}, + {"20240101110000-clean", "customer", "Success", "20240103000000"}, + {"20240101100000-clean", "Customer", "Success", "20240103000000"}, + {"20240101090000-clean", `"quoted db"`, "Success", "20240103000000"}, + {"20240101080000-clean", "customer's db", "Success", "20240103000000"}, + } { + _, err = historyDB.Exec(`INSERT INTO backups (timestamp, database_name, status, date_deleted) VALUES (?, ?, ?, ?)`, backup[0], backup[1], backup[2], backup[3]) + Expect(err).NotTo(HaveOccurred()) + } + }) + + AfterEach(func() { + Expect(historyDB.Close()).To(Succeed()) + }) + + It("adds a bound database predicate only when a database is supplied", func() { + unfiltered := getBackupNameBeforeTimestampQuery("20240101120000", "") + filtered := getBackupNameBeforeTimestampQuery("20240101120000", "customer's db") + Expect(unfiltered).NotTo(ContainSubstring("database_name")) + Expect(filtered).To(ContainSubstring("AND database_name = ?")) + Expect(filtered).NotTo(ContainSubstring("customer's db")) + }) + + It("returns a scan error from a backup-name query", func() { + _, err := execQueryFunc("SELECT NULL", historyDB) + Expect(err).To(HaveOccurred()) + }) + + DescribeTable("returns only exact database matches while retaining unfiltered selection", + func(query func(string, string, *sql.DB) ([]string, error), timestamp string, expectedAll []string, exact, quoted, apostrophe string) { + for database, expected := range map[string][]string{ + "": expectedAll, + "customer": {exact}, + "CUSTOMER": nil, + `"quoted db"`: {quoted}, + "customer's db": {apostrophe}, + "does-not-exist": nil, + } { + actual, err := query(timestamp, database, historyDB) + Expect(err).NotTo(HaveOccurred(), database) + Expect(actual).To(Equal(expected), database) + } + }, + Entry("before timestamp", GetBackupNamesBeforeTimestamp, "20240101120000", + []string{"20240101110000", "20240101100000", "20240101090000", "20240101080000"}, + "20240101110000", "20240101090000", "20240101080000"), + Entry("after timestamp", GetBackupNamesAfterTimestamp, "20240101120000", + []string{"20240102110000", "20240102100000", "20240102090000", "20240102080000"}, + "20240102110000", "20240102090000", "20240102080000"), + Entry("history clean", GetBackupNamesForCleanBeforeTimestamp, "20240101120000", + []string{"20240101110000-clean", "20240101100000-clean", "20240101090000-clean", "20240101080000-clean"}, + "20240101110000-clean", "20240101090000-clean", "20240101080000-clean"), + ) + }) + Describe("deleteBackupsFormTableQuery", func() { It("returns correct query", func() { got := deleteBackupsFormTableQuery("TestBackup", "'20220401102430', '20220401102430'")