SK-3061: close tokenize patch-coverage gaps flagged on #420

.cspell.json

@@ -108,6 +108,8 @@
     "synthesised",
     "deserialise",
     "deserialised",
+    "deserialises",
+    "unparseable",
     "unmodelled",
     "recordss",
     "rarr",

flowvault/src/test/java/com/skyflow/utils/FlatTokenizeResponseTests.java

@@ -338,6 +338,77 @@ public void testRejectedRequest_withUnfamiliarBodyFallsBackToTheStatusCode() {
         Assert.assertEquals(Integer.valueOf(504), records.get(0).getHttpCode());
     }
 
+    @Test
+    public void testRejectedRequest_emptyResponseArrayFallsBackToTheStatusCode() {
+        // "response" is present but empty - nothing to rebuild from, so fall back like a body
+        // without a response array at all
+        List<BulkTokenizeRequestRecord> sent = Collections.singletonList(record("v1", "g1"));
+        Map<String, Object> body = new LinkedHashMap<>();
+        body.put("response", new ArrayList<>());
+
+        List<BulkTokenizeResponseRecord> records =
+                Utils.handleBulkTokenizeBatchException(rejected(500, body), sent, 0);
+
+        Assert.assertEquals(1, records.size());
+        Assert.assertEquals(Integer.valueOf(500), records.get(0).getHttpCode());
+    }
+
+    @Test
+    public void testRejectedRequest_explicitNullResponseArrayFallsBackToTheStatusCode() {
+        // "response" is present in the map but its value is JSON null, not an array - deserialises
+        // to an absent Optional rather than an empty one
+        List<BulkTokenizeRequestRecord> sent = Collections.singletonList(record("v1", "g1"));
+        Map<String, Object> body = new LinkedHashMap<>();
+        body.put("response", null);
+
+        List<BulkTokenizeResponseRecord> records =
+                Utils.handleBulkTokenizeBatchException(rejected(500, body), sent, 0);
+
+        Assert.assertEquals(1, records.size());
+        Assert.assertEquals(Integer.valueOf(500), records.get(0).getHttpCode());
+    }
+
+    @Test
+    public void testRejectedRequest_unparseableResponseArrayFallsBackToTheStatusCode() {
+        // "response" is present but the wrong shape to deserialise - must not propagate the crash
+        List<BulkTokenizeRequestRecord> sent = Collections.singletonList(record("v1", "g1"));
+        Map<String, Object> body = new LinkedHashMap<>();
+        body.put("response", "not-an-array");
+
+        List<BulkTokenizeResponseRecord> records =
+                Utils.handleBulkTokenizeBatchException(rejected(500, body), sent, 0);
+
+        Assert.assertEquals(1, records.size());
+        Assert.assertEquals(Integer.valueOf(500), records.get(0).getHttpCode());
+    }
+
+    @Test
+    public void testRejectedRequest_nullBatchWithPerRowBodyStillRebuildsFromTheRows() {
+        // defensive: a null batch can't be correlated against, but a per-row body still has
+        // everything needed to report each row directly
+        Throwable ex = rejected(400, body(row("v1", "g1", "", "bad group", 400)));
+
+        List<BulkTokenizeResponseRecord> records =
+                Utils.handleBulkTokenizeBatchException(ex, null, 0);
+
+        Assert.assertEquals(1, records.size());
+        Assert.assertEquals(0, records.get(0).getIndex());
+        Assert.assertEquals("bad group", records.get(0).getError());
+    }
+
+    @Test
+    public void testRejectedRequest_emptyBatchWithPerRowBodyStillRebuildsFromTheRows() {
+        // same as a null batch - an empty one can't be correlated against either
+        Throwable ex = rejected(400, body(row("v1", "g1", "", "bad group", 400)));
+
+        List<BulkTokenizeResponseRecord> records =
+                Utils.handleBulkTokenizeBatchException(ex, new ArrayList<>(), 0);
+
+        Assert.assertEquals(1, records.size());
+        Assert.assertEquals(0, records.get(0).getIndex());
+        Assert.assertEquals("bad group", records.get(0).getError());
+    }
+
     @Test
     public void testRejectedRequest_retryableStatusStillSurfacesForRetry() {
         List<BulkTokenizeRequestRecord> sent = Collections.singletonList(record("v1", "g1"));

flowvault/src/test/java/com/skyflow/utils/UtilsTests.java

@@ -1650,6 +1650,39 @@ public void testHandleBulkTokenizeBatchException_errorFieldAsObjectUsesStructure
         Assert.assertEquals(Integer.valueOf(404), errors.get(0).getHttpCode());
     }
 
+    @Test
+    public void testHandleBulkTokenizeBatchException_errorFieldAsObjectPrefersNestedErrorOverMessage() {
+        // extractBatchErrorMessage prefers a nested "error" key over "message" when both are present
+        Map<String, Object> errorObject = new HashMap<>();
+        errorObject.put("error", "nested error message");
+        errorObject.put("message", "vault not found");
+        Map<String, Object> body = new HashMap<>();
+        body.put("error", errorObject);
+        ApiClientApiException apiEx = new ApiClientApiException("tokenize failed", 404, body);
+        RuntimeException wrapper = new RuntimeException(apiEx);
+
+        List<BulkTokenizeResponseRecord> errors = Utils.handleBulkTokenizeBatchException(
+                wrapper, tokenizeBatch("v1", "group1"), 0);
+
+        Assert.assertEquals("nested error message", errors.get(0).getError());
+    }
+
+    @Test
+    public void testHandleBulkTokenizeBatchException_errorFieldAsObjectWithoutAStringFallsBackToApiMessage() {
+        // neither "error" nor "message" is a String, so there is nothing usable to read out of it
+        Map<String, Object> errorObject = new HashMap<>();
+        errorObject.put("message", Collections.singletonList("not a string"));
+        Map<String, Object> body = new HashMap<>();
+        body.put("error", errorObject);
+        ApiClientApiException apiEx = new ApiClientApiException("tokenize failed", 404, body);
+        RuntimeException wrapper = new RuntimeException(apiEx);
+
+        List<BulkTokenizeResponseRecord> errors = Utils.handleBulkTokenizeBatchException(
+                wrapper, tokenizeBatch("v1", "group1"), 0);
+
+        Assert.assertEquals("tokenize failed", errors.get(0).getError());
+    }
+
     @Test
     public void testHandleBulkTokenizeBatchException_nonMapBodyUsesApiMessage() {
         // Body is not a map, so extractBatchErrorMessage falls back to the exception's own message.
@@ -1673,6 +1706,19 @@ public void testHandleBulkTokenizeBatchException_nullBatchReturnsEmpty() {
         Assert.assertTrue(errors.isEmpty());
     }
 
+    @Test
+    public void testHandleBulkTokenizeBatchException_emptyGroupListStillReportsOneEntry() {
+        // an explicitly empty token group list, not a null one, must be treated the same way
+        RuntimeException ex = new RuntimeException("boom");
+        List<BulkTokenizeRequestRecord> batch = Collections.singletonList(
+                BulkTokenizeRequestRecord.builder().value("v1").tokenGroupNames(new ArrayList<>()).build());
+
+        List<BulkTokenizeResponseRecord> errors = Utils.handleBulkTokenizeBatchException(ex, batch, 0);
+
+        Assert.assertEquals(1, errors.size());
+        Assert.assertNull(errors.get(0).getTokenGroupName());
+    }
+
     // ── formatBulkInsertResponse ───────────────────────────────────────────────
 
     @Test
@@ -1979,6 +2025,12 @@ public void testFormatBulkTokenizeResponse_emptyResponseReturnsNull() {
                 tokenizeBatch("value1", "group1"), 0, new HashMap<>()));
     }
 
+    @Test
+    public void testFormatBulkTokenizeResponse_nullResponseReturnsNull() {
+        Assert.assertNull(Utils.formatBulkTokenizeResponse(
+                null, tokenizeBatch("value1", "group1"), 0, new HashMap<>()));
+    }
+
     // Tests for getQueryRequestBody / buildQueryResponse / getGetRequestBody / buildGetResponse
     // were removed: get and query Utils helpers no longer exist (bulk-only module).
 

flowvault/src/test/java/com/skyflow/vault/data/BulkRetryAndSummaryTests.java

@@ -108,8 +108,43 @@ public void testTokenizeSummary_isNullWhenNoPayloadWasSupplied() {
         Assert.assertNull(new BulkTokenizeResponse(new ArrayList<>()).getSummary());
     }
 
+    @Test
+    public void testTokenizeSummary_classifiesByRowsWhenNoPayloadIsGiven() {
+        // The two-arg constructor can still be called with a null payload; classification then
+        // falls back to the indexes actually present in records instead of the submitted list.
+        List<BulkTokenizeResponseRecord> records = Arrays.asList(
+                row(0, "g1", "t1", 200, null),
+                row(1, "g1", null, 400, "bad group"),
+                row(2, "g1", "t2", 200, null),
+                row(2, "g2", null, 400, "bad group"));
+
+        TokenizeSummary summary = new BulkTokenizeResponse(records, null).getSummary();
+
+        // without a submitted payload to count values from, totalTokens falls back to the row count
+        Assert.assertEquals(4, summary.getTotalTokens());
+        Assert.assertEquals(1, summary.getTotalTokenized());
+        Assert.assertEquals(1, summary.getTotalPartial());
+        Assert.assertEquals(1, summary.getTotalFailed());
+    }
+
+    @Test
+    public void testTokenizeSummary_nullRecordsAndNullPayloadYieldsZeroes() {
+        TokenizeSummary summary = new BulkTokenizeResponse(null, null).getSummary();
+
+        Assert.assertEquals(0, summary.getTotalTokens());
+        Assert.assertEquals(0, summary.getTotalTokenized());
+        Assert.assertEquals(0, summary.getTotalPartial());
+        Assert.assertEquals(0, summary.getTotalFailed());
+    }
+
     // ── BulkTokenizeResponse.getRecordsToRetry ────────────────────────────────
 
+    @Test
+    public void testTokenizeRetry_withNullRecordsReturnsEmpty() {
+        Assert.assertTrue(new BulkTokenizeResponse(null, Collections.singletonList(requestRecord("a")))
+                .getRecordsToRetry().isEmpty());
+    }
+
     @Test
     public void testTokenizeRetry_only5xxFailuresAreReturned() {
         List<BulkTokenizeResponseRecord> records = Arrays.asList(