HDDS-16420. Use partial cache for OmLifecycleConfiguration and OmLifecycleScanState table

hadoop-ozone/interface-storage/src/main/java/org/apache/hadoop/ozone/om/OMMetadataManager.java

@@ -38,7 +38,6 @@
 import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
 import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
 import org.apache.hadoop.ozone.common.BlockGroup;
-import org.apache.hadoop.ozone.om.exceptions.OMException;
 import org.apache.hadoop.ozone.om.helpers.BucketLayout;
 import org.apache.hadoop.ozone.om.helpers.ListKeysResult;
 import org.apache.hadoop.ozone.om.helpers.ListOpenFilesResult;
@@ -510,7 +509,7 @@ String getMultipartKeyFSO(String volume, String bucket, String key, String
   /**
    * @return list all LifecycleConfigurations.
    */
-  List<OmLifecycleConfiguration> listLifecycleConfigurations() throws OMException;
+  List<OmLifecycleConfiguration> listLifecycleConfigurations() throws IOException;
 
   /**
    * Fetches the lifecycle configuration by bucketName.

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OmMetadataManagerImpl.java

@@ -546,8 +546,8 @@ protected void initializeOmTables(CacheType cacheType,
     s3RevokedStsTokenTable = initializer.get(
         OMDBDefinition.S3_REVOKED_STS_TOKEN_TABLE_DEF, cacheType);
 
-    lifecycleConfigurationTable = initializer.get(OMDBDefinition.LIFECYCLE_CONFIGURATION_TABLE_DEF, cacheType);
-    lifecycleScanStateTable = initializer.get(OMDBDefinition.LIFECYCLE_SCAN_STATE_TABLE_DEF, cacheType);
+    lifecycleConfigurationTable = initializer.get(OMDBDefinition.LIFECYCLE_CONFIGURATION_TABLE_DEF);
+    lifecycleScanStateTable = initializer.get(OMDBDefinition.LIFECYCLE_SCAN_STATE_TABLE_DEF);
   }
 
   /**
@@ -1779,25 +1779,39 @@ public Table<String, OmLifecycleScanState> getLifecycleScanStateTable() {
    * @return list all LifecycleConfigurations.
    */
   @Override
-  public List<OmLifecycleConfiguration> listLifecycleConfigurations() {
+  public List<OmLifecycleConfiguration> listLifecycleConfigurations() throws IOException {
     List<OmLifecycleConfiguration> result = Lists.newArrayList();
+    Set<String> cachedKeys = new HashSet<>();
 
-    /* lifecycleConfigurationTable is full-cache, so we use cacheIterator. */
+    // lifecycleConfigurationTable uses partial cache, so cacheIterator() only returns
+    // entries that are currently in memory. Process cache entries first to handle
+    // any pending writes or pending deletes that have not yet been flushed to RocksDB.
     Iterator<Map.Entry<CacheKey<String>, CacheValue<OmLifecycleConfiguration>>>
         cacheIterator = getLifecycleConfigurationTable().cacheIterator();
-
-    OmLifecycleConfiguration lifecycleConfiguration;
     while (cacheIterator.hasNext()) {
       Map.Entry<CacheKey<String>, CacheValue<OmLifecycleConfiguration>> entry =
           cacheIterator.next();
-      lifecycleConfiguration = entry.getValue().getCacheValue();
+      cachedKeys.add(entry.getKey().getCacheKey());
+      OmLifecycleConfiguration lifecycleConfiguration = entry.getValue().getCacheValue();
       if (lifecycleConfiguration == null) {
-        // lifecycleConfiguration null means it's a deleted.
+        // null means it's a pending delete.
         continue;
       }
       result.add(lifecycleConfiguration);
     }
 
+    // Also iterate RocksDB to pick up entries that have been evicted from (or were
+    // never loaded into) the partial cache.
+    try (TableIterator<String, ? extends KeyValue<String, OmLifecycleConfiguration>>
+             iter = getLifecycleConfigurationTable().iterator()) {
+      while (iter.hasNext()) {
+        KeyValue<String, OmLifecycleConfiguration> kv = iter.next();
+        if (!cachedKeys.contains(kv.getKey())) {
+          result.add(kv.getValue());
+        }
+      }
+    }
+
     return result;
   }
 

hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/KeyLifecycleService.java

@@ -202,7 +202,7 @@ public BackgroundTaskQueue getTasks() {
     List<OmLifecycleConfiguration> lifecycleConfigurationList = null;
     try {
       lifecycleConfigurationList = omMetadataManager.listLifecycleConfigurations();
-    } catch (OMException e) {
+    } catch (IOException e) {
       LOG.error("Failed to list lifecycle configurations", e);
       return queue;
     }