From ca332dd79be3aaca51f4cd77b1a14b35eff7ebc6 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Tue, 15 Sep 2026 13:15:38 +0800 Subject: [PATCH 1/4] HDDS-16420. Use partial cache for OmLifecycleConfiguration and OmLifecycleScanState table --- .../org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index ecbfc7ac395..9f84e8f3622 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -546,8 +546,8 @@ protected void initializeOmTables(CacheType cacheType, s3RevokedStsTokenTable = initializer.get( OMDBDefinition.S3_REVOKED_STS_TOKEN_TABLE_DEF, cacheType); - lifecycleConfigurationTable = initializer.get(OMDBDefinition.LIFECYCLE_CONFIGURATION_TABLE_DEF, cacheType); - lifecycleScanStateTable = initializer.get(OMDBDefinition.LIFECYCLE_SCAN_STATE_TABLE_DEF, cacheType); + lifecycleConfigurationTable = initializer.get(OMDBDefinition.LIFECYCLE_CONFIGURATION_TABLE_DEF); + lifecycleScanStateTable = initializer.get(OMDBDefinition.LIFECYCLE_SCAN_STATE_TABLE_DEF); } /** From 5204a0f2326ae8bf3d1f8a5bfebc9ae7ccf4d3b6 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Tue, 15 Sep 2026 19:56:00 +0800 Subject: [PATCH 2/4] update listLifecycleConfigurations implementation --- .../hadoop/ozone/om/OMMetadataManager.java | 2 +- .../ozone/om/OmMetadataManagerImpl.java | 26 ++++++++++++++----- 2 files changed, 21 insertions(+), 7 deletions(-) diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index 37059ba4600..e34eedea15c 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -510,7 +510,7 @@ String getMultipartKeyFSO(String volume, String bucket, String key, String /** * @return list all LifecycleConfigurations. */ - List listLifecycleConfigurations() throws OMException; + List listLifecycleConfigurations() throws IOException; /** * Fetches the lifecycle configuration by bucketName. diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java index 9f84e8f3622..dc0bba895eb 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java @@ -1779,25 +1779,39 @@ public Table getLifecycleScanStateTable() { * @return list all LifecycleConfigurations. */ @Override - public List listLifecycleConfigurations() { + public List listLifecycleConfigurations() throws IOException { List result = Lists.newArrayList(); + Set cachedKeys = new HashSet<>(); - /* lifecycleConfigurationTable is full-cache, so we use cacheIterator. */ + // lifecycleConfigurationTable uses partial cache, so cacheIterator() only returns + // entries that are currently in memory. Process cache entries first to handle + // any pending writes or pending deletes that have not yet been flushed to RocksDB. Iterator, CacheValue>> cacheIterator = getLifecycleConfigurationTable().cacheIterator(); - - OmLifecycleConfiguration lifecycleConfiguration; while (cacheIterator.hasNext()) { Map.Entry, CacheValue> entry = cacheIterator.next(); - lifecycleConfiguration = entry.getValue().getCacheValue(); + cachedKeys.add(entry.getKey().getCacheKey()); + OmLifecycleConfiguration lifecycleConfiguration = entry.getValue().getCacheValue(); if (lifecycleConfiguration == null) { - // lifecycleConfiguration null means it's a deleted. + // null means it's a pending delete. continue; } result.add(lifecycleConfiguration); } + // Also iterate RocksDB to pick up entries that have been evicted from (or were + // never loaded into) the partial cache. + try (TableIterator> + iter = getLifecycleConfigurationTable().iterator()) { + while (iter.hasNext()) { + KeyValue kv = iter.next(); + if (!cachedKeys.contains(kv.getKey())) { + result.add(kv.getValue()); + } + } + } + return result; } From eea53953caadec7d9630b5794c9e93dc63aa2ea2 Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Tue, 15 Sep 2026 20:04:05 +0800 Subject: [PATCH 3/4] remove unused import --- .../main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java | 1 - 1 file changed, 1 deletion(-) diff --git a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java index e34eedea15c..20f5949f1a0 100644 --- a/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java +++ b/hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java @@ -38,7 +38,6 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.common.BlockGroup; -import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.ListKeysResult; import org.apache.hadoop.ozone.om.helpers.ListOpenFilesResult; From 6d4140d6045e63508c80d566a025a9b20e8673ae Mon Sep 17 00:00:00 2001 From: Sammi Chen Date: Tue, 15 Sep 2026 21:03:44 +0800 Subject: [PATCH 4/4] fix pmd error --- .../org/apache/hadoop/ozone/om/service/KeyLifecycleService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java index f3c86851a85..ade0b0d72f6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java @@ -202,7 +202,7 @@ public BackgroundTaskQueue getTasks() { List lifecycleConfigurationList = null; try { lifecycleConfigurationList = omMetadataManager.listLifecycleConfigurations(); - } catch (OMException e) { + } catch (IOException e) { LOG.error("Failed to list lifecycle configurations", e); return queue; }