From 5ecf1a5954dd7e8ec68cf123fb8d9e0c86bc88e0 Mon Sep 17 00:00:00 2001 From: Starfolk Date: Fri, 4 Sep 2026 19:57:32 +0000 Subject: [PATCH 1/3] test(litellm): parametrize sync/async test pairs Collapse 13 duplicated sync/async LiteLLM test pairs into single parametrized tests using a shared `sync_async` marker plus two small helpers (`_call` and `_collect_stream`). Cuts test_litellm.py from 1088 to ~828 lines with no loss of coverage. Where the async variant historically asserted less than its sync twin, the unified body now applies the stronger assertions to both (verified against the existing cassettes): streaming content/output checks, rerank index/output-shape/search_units, and the litellm-auto span origin. Cassettes are renamed to the parametrized node names (e.g. test_litellm_acompletion_metrics.yaml -> test_litellm_completion_metrics[async].yaml) across both the latest and 1.74.0 matrix dirs. Pure git renames, no recorded content changed. Validated: nox -s "test_litellm(latest)" and "test_litellm(1.74.0)" both pass 38 tests; pylint (--errors-only) and pre-commit clean. Co-Authored-By: Claude Opus 4.8 --- ...st_litellm_completion_metrics[async].yaml} | 0 ...est_litellm_completion_metrics[sync].yaml} | 0 ..._litellm_completion_streaming[async].yaml} | 0 ...t_litellm_completion_streaming[sync].yaml} | 0 ...completion_with_system_prompt[async].yaml} | 0 ..._completion_with_system_prompt[sync].yaml} | 0 ...aml => test_litellm_embedding[async].yaml} | 0 ...yaml => test_litellm_embedding[sync].yaml} | 0 ...test_litellm_image_generation[async].yaml} | 0 ... test_litellm_image_generation[sync].yaml} | 0 ...ml => test_litellm_moderation[async].yaml} | 0 ...aml => test_litellm_moderation[sync].yaml} | 0 ...k.yaml => test_litellm_rerank[async].yaml} | 0 ...nk.yaml => test_litellm_rerank[sync].yaml} | 0 ...est_litellm_responses_metrics[async].yaml} | 0 ...test_litellm_responses_metrics[sync].yaml} | 0 ...t_litellm_responses_streaming[async].yaml} | 0 ...st_litellm_responses_streaming[sync].yaml} | 0 ...h.yaml => test_litellm_speech[async].yaml} | 0 ...ch.yaml => test_litellm_speech[sync].yaml} | 0 ...tellm_text_completion_metrics[async].yaml} | 0 ...itellm_text_completion_metrics[sync].yaml} | 0 ...=> test_litellm_transcription[async].yaml} | 0 ... => test_litellm_transcription[sync].yaml} | 0 ...st_litellm_completion_metrics[async].yaml} | 0 ...est_litellm_completion_metrics[sync].yaml} | 0 ..._litellm_completion_streaming[async].yaml} | 0 ...t_litellm_completion_streaming[sync].yaml} | 0 ...completion_with_system_prompt[async].yaml} | 0 ..._completion_with_system_prompt[sync].yaml} | 0 ...aml => test_litellm_embedding[async].yaml} | 0 ...yaml => test_litellm_embedding[sync].yaml} | 0 ...test_litellm_image_generation[async].yaml} | 0 ... test_litellm_image_generation[sync].yaml} | 0 ...ml => test_litellm_moderation[async].yaml} | 0 ...aml => test_litellm_moderation[sync].yaml} | 0 ...k.yaml => test_litellm_rerank[async].yaml} | 0 ...nk.yaml => test_litellm_rerank[sync].yaml} | 0 ...est_litellm_responses_metrics[async].yaml} | 0 ...test_litellm_responses_metrics[sync].yaml} | 0 ...t_litellm_responses_streaming[async].yaml} | 0 ...st_litellm_responses_streaming[sync].yaml} | 0 ...h.yaml => test_litellm_speech[async].yaml} | 0 ...ch.yaml => test_litellm_speech[sync].yaml} | 0 ...tellm_text_completion_metrics[async].yaml} | 0 ...itellm_text_completion_metrics[sync].yaml} | 0 ...=> test_litellm_transcription[async].yaml} | 0 ... => test_litellm_transcription[sync].yaml} | 0 .../integrations/litellm/test_litellm.py | 490 +++++------------- 49 files changed, 124 insertions(+), 366 deletions(-) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_acompletion_metrics.yaml => test_litellm_completion_metrics[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_completion_metrics.yaml => test_litellm_completion_metrics[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_acompletion_streaming_async.yaml => test_litellm_completion_streaming[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_completion_streaming_sync.yaml => test_litellm_completion_streaming[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_acompletion_with_system_prompt.yaml => test_litellm_completion_with_system_prompt[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_completion_with_system_prompt.yaml => test_litellm_completion_with_system_prompt[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_aembedding.yaml => test_litellm_embedding[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_embeddings.yaml => test_litellm_embedding[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_aimage_generation.yaml => test_litellm_image_generation[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_image_generation.yaml => test_litellm_image_generation[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_amoderation.yaml => test_litellm_moderation[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_moderation.yaml => test_litellm_moderation[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_arerank.yaml => test_litellm_rerank[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_rerank.yaml => test_litellm_rerank[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_aresponses_metrics.yaml => test_litellm_responses_metrics[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_responses_metrics.yaml => test_litellm_responses_metrics[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_aresponses_streaming_async.yaml => test_litellm_responses_streaming[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_responses_streaming_sync.yaml => test_litellm_responses_streaming[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_aspeech.yaml => test_litellm_speech[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_speech.yaml => test_litellm_speech[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_atext_completion_metrics.yaml => test_litellm_text_completion_metrics[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_text_completion_metrics.yaml => test_litellm_text_completion_metrics[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_atranscription.yaml => test_litellm_transcription[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/1.74.0/{test_litellm_transcription.yaml => test_litellm_transcription[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_acompletion_metrics.yaml => test_litellm_completion_metrics[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_completion_metrics.yaml => test_litellm_completion_metrics[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_acompletion_streaming_async.yaml => test_litellm_completion_streaming[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_completion_streaming_sync.yaml => test_litellm_completion_streaming[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_acompletion_with_system_prompt.yaml => test_litellm_completion_with_system_prompt[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_completion_with_system_prompt.yaml => test_litellm_completion_with_system_prompt[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_aembedding.yaml => test_litellm_embedding[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_embeddings.yaml => test_litellm_embedding[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_aimage_generation.yaml => test_litellm_image_generation[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_image_generation.yaml => test_litellm_image_generation[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_amoderation.yaml => test_litellm_moderation[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_moderation.yaml => test_litellm_moderation[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_arerank.yaml => test_litellm_rerank[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_rerank.yaml => test_litellm_rerank[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_aresponses_metrics.yaml => test_litellm_responses_metrics[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_responses_metrics.yaml => test_litellm_responses_metrics[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_aresponses_streaming_async.yaml => test_litellm_responses_streaming[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_responses_streaming_sync.yaml => test_litellm_responses_streaming[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_aspeech.yaml => test_litellm_speech[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_speech.yaml => test_litellm_speech[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_atext_completion_metrics.yaml => test_litellm_text_completion_metrics[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_text_completion_metrics.yaml => test_litellm_text_completion_metrics[sync].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_atranscription.yaml => test_litellm_transcription[async].yaml} (100%) rename py/src/braintrust/integrations/litellm/cassettes/latest/{test_litellm_transcription.yaml => test_litellm_transcription[sync].yaml} (100%) diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_acompletion_metrics.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_acompletion_metrics.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_metrics[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_acompletion_streaming_async.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_acompletion_streaming_async.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming_sync.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming_sync.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_streaming[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_acompletion_with_system_prompt.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_with_system_prompt[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_acompletion_with_system_prompt.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_with_system_prompt[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_with_system_prompt.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_with_system_prompt[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_with_system_prompt.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_completion_with_system_prompt[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_aembedding.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_embedding[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_aembedding.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_embedding[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_embeddings.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_embedding[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_embeddings.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_embedding[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_aimage_generation.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_generation[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_aimage_generation.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_generation[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_generation.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_generation[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_generation.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_image_generation[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_amoderation.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_moderation[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_amoderation.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_moderation[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_moderation.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_moderation[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_moderation.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_moderation[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_arerank.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_rerank[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_arerank.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_rerank[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_rerank.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_rerank[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_rerank.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_rerank[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_aresponses_metrics.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_responses_metrics[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_aresponses_metrics.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_responses_metrics[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_responses_metrics.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_responses_metrics[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_responses_metrics.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_responses_metrics[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_aresponses_streaming_async.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_responses_streaming[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_aresponses_streaming_async.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_responses_streaming[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_responses_streaming_sync.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_responses_streaming[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_responses_streaming_sync.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_responses_streaming[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_aspeech.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_speech[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_aspeech.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_speech[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_speech.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_speech[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_speech.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_speech[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_atext_completion_metrics.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_text_completion_metrics[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_atext_completion_metrics.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_text_completion_metrics[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_text_completion_metrics.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_text_completion_metrics[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_text_completion_metrics.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_text_completion_metrics[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_atranscription.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_transcription[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_atranscription.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_transcription[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_transcription.yaml b/py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_transcription[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_transcription.yaml rename to py/src/braintrust/integrations/litellm/cassettes/1.74.0/test_litellm_transcription[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_acompletion_metrics.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_acompletion_metrics.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_metrics[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_acompletion_streaming_async.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_acompletion_streaming_async.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming_sync.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming_sync.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_streaming[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_acompletion_with_system_prompt.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_with_system_prompt[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_acompletion_with_system_prompt.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_with_system_prompt[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_with_system_prompt.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_with_system_prompt[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_with_system_prompt.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_completion_with_system_prompt[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_aembedding.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_embedding[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_aembedding.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_embedding[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_embeddings.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_embedding[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_embeddings.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_embedding[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_aimage_generation.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_generation[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_aimage_generation.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_generation[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_generation.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_generation[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_generation.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_image_generation[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_amoderation.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_moderation[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_amoderation.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_moderation[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_moderation.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_moderation[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_moderation.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_moderation[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_arerank.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_rerank[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_arerank.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_rerank[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_rerank.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_rerank[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_rerank.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_rerank[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_aresponses_metrics.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_responses_metrics[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_aresponses_metrics.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_responses_metrics[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_responses_metrics.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_responses_metrics[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_responses_metrics.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_responses_metrics[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_aresponses_streaming_async.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_responses_streaming[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_aresponses_streaming_async.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_responses_streaming[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_responses_streaming_sync.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_responses_streaming[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_responses_streaming_sync.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_responses_streaming[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_aspeech.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_speech[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_aspeech.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_speech[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_speech.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_speech[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_speech.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_speech[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_atext_completion_metrics.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_text_completion_metrics[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_atext_completion_metrics.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_text_completion_metrics[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_text_completion_metrics.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_text_completion_metrics[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_text_completion_metrics.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_text_completion_metrics[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_atranscription.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_transcription[async].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_atranscription.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_transcription[async].yaml diff --git a/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_transcription.yaml b/py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_transcription[sync].yaml similarity index 100% rename from py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_transcription.yaml rename to py/src/braintrust/integrations/litellm/cassettes/latest/test_litellm_transcription[sync].yaml diff --git a/py/src/braintrust/integrations/litellm/test_litellm.py b/py/src/braintrust/integrations/litellm/test_litellm.py index 373dc8113..29a1ca21c 100644 --- a/py/src/braintrust/integrations/litellm/test_litellm.py +++ b/py/src/braintrust/integrations/litellm/test_litellm.py @@ -32,6 +32,31 @@ "Madrid is the capital of Spain.", ] +# Applied to every sync/async pair below. The ``is_async`` param selects the +# async LiteLLM entrypoint (e.g. ``acompletion``) over the sync one; the two +# variants share a single test body and record to ``...[sync]`` / ``...[async]`` +# cassettes. +sync_async = pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) + + +async def _call(is_async, sync_fn, async_fn, *args, **kwargs): + """Invoke ``async_fn`` (awaited) when ``is_async`` else ``sync_fn``.""" + if is_async: + return await async_fn(*args, **kwargs) + return sync_fn(*args, **kwargs) + + +async def _collect_stream(stream, is_async): + """Drain a sync or async LiteLLM stream into a list of chunks.""" + chunks = [] + if is_async: + async for chunk in stream: + chunks.append(chunk) + else: + for chunk in stream: + chunks.append(chunk) + return chunks + def _assert_speech_output_attachment(span) -> None: assert span["output"]["type"] == "audio" @@ -55,11 +80,19 @@ def memory_logger(): @pytest.mark.vcr -def test_litellm_completion_metrics(memory_logger) -> None: +@pytest.mark.asyncio +@sync_async +async def test_litellm_completion_metrics(memory_logger, is_async) -> None: assert not memory_logger.pop() start = time.time() - response = litellm.completion(model=TEST_MODEL, messages=[{"role": "user", "content": TEST_PROMPT}]) + response = await _call( + is_async, + litellm.completion, + litellm.acompletion, + model=TEST_MODEL, + messages=[{"role": "user", "content": TEST_PROMPT}], + ) end = time.time() assert response @@ -153,34 +186,18 @@ def test_litellm_prompt_caching_metrics(memory_logger) -> None: @pytest.mark.vcr @pytest.mark.asyncio -async def test_litellm_acompletion_metrics(memory_logger): +@sync_async +async def test_litellm_text_completion_metrics(memory_logger, is_async) -> None: assert not memory_logger.pop() start = time.time() - response = await litellm.acompletion(model=TEST_MODEL, messages=[{"role": "user", "content": TEST_PROMPT}]) - end = time.time() - - assert response - assert response.choices[0].message.content - assert "24" in response.choices[0].message.content or "twenty-four" in response.choices[0].message.content.lower() - - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - assert span - metrics = span["metrics"] - assert_metrics_are_valid(metrics, start, end) - assert span["metadata"]["model"] == TEST_MODEL - assert span["metadata"]["provider"] == "openai" - assert TEST_PROMPT in str(span["input"]) - - -@pytest.mark.vcr -def test_litellm_text_completion_metrics(memory_logger) -> None: - assert not memory_logger.pop() - - start = time.time() - response = litellm.text_completion(model=TEST_TEXT_MODEL, prompt=TEST_PROMPT) + response = await _call( + is_async, + litellm.text_completion, + litellm.atext_completion, + model=TEST_TEXT_MODEL, + prompt=TEST_PROMPT, + ) end = time.time() assert response @@ -201,43 +218,20 @@ def test_litellm_text_completion_metrics(memory_logger) -> None: @pytest.mark.vcr @pytest.mark.asyncio -async def test_litellm_atext_completion_metrics(memory_logger): +@sync_async +async def test_litellm_completion_streaming(memory_logger, is_async): assert not memory_logger.pop() start = time.time() - response = await litellm.atext_completion(model=TEST_TEXT_MODEL, prompt=TEST_PROMPT) - end = time.time() - - assert response - assert response.choices[0].text - assert "24" in response.choices[0].text or "twenty-four" in response.choices[0].text.lower() - - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - assert span - metrics = span["metrics"] - assert_metrics_are_valid(metrics, start, end) - assert span["metadata"]["model"] == TEST_TEXT_MODEL - assert span["metadata"]["provider"] == "openai" - assert TEST_PROMPT in str(span["input"]) - assert "text" in span["output"][0] - - -@pytest.mark.vcr -def test_litellm_completion_streaming_sync(memory_logger): - assert not memory_logger.pop() - - start = time.time() - stream = litellm.completion( + stream = await _call( + is_async, + litellm.completion, + litellm.acompletion, model=TEST_MODEL, messages=[{"role": "user", "content": TEST_PROMPT}], stream=True, ) - - chunks = [] - for chunk in stream: - chunks.append(chunk) + chunks = await _collect_stream(stream, is_async) end = time.time() # Verify streaming works @@ -268,43 +262,15 @@ def test_litellm_completion_streaming_sync(memory_logger): @pytest.mark.vcr @pytest.mark.asyncio -async def test_litellm_acompletion_streaming_async(memory_logger): - assert not memory_logger.pop() - - start = time.time() - stream = await litellm.acompletion( - model=TEST_MODEL, - messages=[{"role": "user", "content": TEST_PROMPT}], - stream=True, - ) - - chunks = [] - async for chunk in stream: - chunks.append(chunk) - end = time.time() - - # Verify streaming works - assert chunks - assert len(chunks) > 1 - - # Verify spans were created - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - assert span - metrics = span["metrics"] - assert_metrics_are_valid(metrics, start, end) - assert span["metadata"]["model"] == TEST_MODEL - assert span["metadata"]["provider"] == "openai" - assert TEST_PROMPT in str(span["input"]) - - -@pytest.mark.vcr -def test_litellm_responses_metrics(memory_logger): +@sync_async +async def test_litellm_responses_metrics(memory_logger, is_async): assert not memory_logger.pop() start = time.time() - response = litellm.responses( + response = await _call( + is_async, + litellm.responses, + litellm.aresponses, model=TEST_MODEL, input=TEST_PROMPT, instructions="Just the number please", @@ -331,42 +297,17 @@ def test_litellm_responses_metrics(memory_logger): @pytest.mark.vcr @pytest.mark.asyncio -async def test_litellm_aresponses_metrics(memory_logger): +@sync_async +async def test_litellm_embedding(memory_logger, is_async): assert not memory_logger.pop() - start = time.time() - response = await litellm.aresponses( - model=TEST_MODEL, - input=TEST_PROMPT, - instructions="Just the number please", + response = await _call( + is_async, + litellm.embedding, + litellm.aembedding, + model="text-embedding-ada-002", + input="This is a test", ) - end = time.time() - - assert response - assert response.output - assert len(response.output) > 0 - content = response.output[0].content[0].text - assert "24" in content or "twenty-four" in content.lower() - - # Verify spans were created - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - assert span - metrics = span["metrics"] - assert_metrics_are_valid(metrics, start, end) - assert span["metadata"]["model"] == TEST_MODEL - assert span["metadata"]["provider"] == "openai" - assert TEST_PROMPT in str(span["input"]) - - -@pytest.mark.vcr -def test_litellm_embeddings(memory_logger): - assert not memory_logger.pop() - - start = time.time() - response = litellm.embedding(model="text-embedding-ada-002", input="This is a test") - end = time.time() assert response assert response.data @@ -384,31 +325,17 @@ def test_litellm_embeddings(memory_logger): @pytest.mark.vcr @pytest.mark.asyncio -async def test_litellm_aembedding(memory_logger): +@sync_async +async def test_litellm_moderation(memory_logger, is_async): assert not memory_logger.pop() - response = await litellm.aembedding(model="text-embedding-ada-002", input="This is a test") - - assert response - assert response.data - assert response.data[0]["embedding"] - - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - assert span - assert span["metadata"]["model"] == "text-embedding-ada-002" - assert span["metadata"]["provider"] == "openai" - assert "This is a test" in str(span["input"]) - - -@pytest.mark.vcr -def test_litellm_moderation(memory_logger): - assert not memory_logger.pop() - - start = time.time() - response = litellm.moderation(model="omni-moderation-latest", input="This is a test message") - end = time.time() + response = await _call( + is_async, + litellm.moderation, + litellm.amoderation, + model="omni-moderation-latest", + input="This is a test message", + ) assert response assert response.results @@ -425,29 +352,16 @@ def test_litellm_moderation(memory_logger): @pytest.mark.vcr @pytest.mark.asyncio -async def test_litellm_amoderation(memory_logger): - assert not memory_logger.pop() - - response = await litellm.amoderation(model="omni-moderation-latest", input="This is a test message") - - assert response - assert response.results - - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - assert span["metadata"]["model"] == "omni-moderation-latest" - assert span["metadata"]["provider"] == "openai" - assert "This is a test message" in str(span["input"]) - - -@pytest.mark.vcr -def test_litellm_image_generation(memory_logger): +@sync_async +async def test_litellm_image_generation(memory_logger, is_async): assert not memory_logger.pop() prompt = "A tiny red square on a white background" - response = litellm.image_generation( + response = await _call( + is_async, + litellm.image_generation, + litellm.aimage_generation, model="gpt-image-1-mini", prompt=prompt, size="1024x1024", @@ -469,36 +383,14 @@ def test_litellm_image_generation(memory_logger): @pytest.mark.vcr @pytest.mark.asyncio -async def test_litellm_aimage_generation(memory_logger): +@sync_async +async def test_litellm_completion_with_system_prompt(memory_logger, is_async): assert not memory_logger.pop() - prompt = "A tiny blue square on a white background" - - response = await litellm.aimage_generation( - model="gpt-image-1-mini", - prompt=prompt, - size="1024x1024", - ) - - assert response - assert response.data - assert response.data[0].b64_json or response.data[0].url - - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - assert span["metadata"]["model"] == "gpt-image-1-mini" - assert span["metadata"]["provider"] == "openai" - assert span["input"] == prompt - assert span["output"]["images_count"] == 1 - assert span["metrics"]["duration"] >= 0 - - -@pytest.mark.vcr -def test_litellm_completion_with_system_prompt(memory_logger): - assert not memory_logger.pop() - - response = litellm.completion( + response = await _call( + is_async, + litellm.completion, + litellm.acompletion, model=TEST_MODEL, messages=[{"role": "system", "content": TEST_SYSTEM_PROMPT}, {"role": "user", "content": TEST_PROMPT}], ) @@ -518,34 +410,14 @@ def test_litellm_completion_with_system_prompt(memory_logger): assert inputs[1]["content"] == TEST_PROMPT -@pytest.mark.vcr -def test_litellm_transcription(memory_logger): - assert not memory_logger.pop() - - with open(TEST_AUDIO_FILE, "rb") as f: - response = litellm.transcription(model="whisper-1", file=f) - - assert response - assert response.text == "you" - - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - assert span["metadata"]["model"] == "whisper-1" - assert span["metadata"]["provider"] == "openai" - assert isinstance(span["input"]["file"], Attachment) - assert span["input"]["file"].reference["filename"] == "test_audio.wav" - assert span["input"]["file"].reference["content_type"] in ("audio/x-wav", "audio/wav") # OS-dependent - assert span["output"] == "you" - - @pytest.mark.vcr @pytest.mark.asyncio -async def test_litellm_atranscription(memory_logger): +@sync_async +async def test_litellm_transcription(memory_logger, is_async): assert not memory_logger.pop() with open(TEST_AUDIO_FILE, "rb") as f: - response = await litellm.atranscription(model="whisper-1", file=f) + response = await _call(is_async, litellm.transcription, litellm.atranscription, model="whisper-1", file=f) assert response assert response.text == "you" @@ -561,37 +433,16 @@ async def test_litellm_atranscription(memory_logger): assert span["output"] == "you" -@pytest.mark.vcr -def test_litellm_speech(memory_logger): - assert not memory_logger.pop() - - response = litellm.speech( - model="tts-1", - voice="alloy", - input="Hello, this is a test.", - response_format="mp3", - ) - - assert response - assert response.content - - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - assert span["metadata"]["model"] == "tts-1" - assert span["metadata"]["voice"] == "alloy" - assert span["metadata"]["response_format"] == "mp3" - assert span["metadata"]["provider"] == "openai" - assert span["input"] == "Hello, this is a test." - _assert_speech_output_attachment(span) - - @pytest.mark.vcr @pytest.mark.asyncio -async def test_litellm_aspeech(memory_logger): +@sync_async +async def test_litellm_speech(memory_logger, is_async): assert not memory_logger.pop() - response = await litellm.aspeech( + response = await _call( + is_async, + litellm.speech, + litellm.aspeech, model="tts-1", voice="alloy", input="Hello, this is a test.", @@ -614,61 +465,21 @@ async def test_litellm_aspeech(memory_logger): @pytest.mark.vcr @pytest.mark.asyncio -async def test_litellm_acompletion_with_system_prompt(memory_logger): - assert not memory_logger.pop() - - response = await litellm.acompletion( - model=TEST_MODEL, - messages=[{"role": "system", "content": TEST_SYSTEM_PROMPT}, {"role": "user", "content": TEST_PROMPT}], - ) - - assert response - assert response.choices - assert "24" in response.choices[0].message.content - - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - inputs = span["input"] - assert len(inputs) == 2 - assert inputs[0]["role"] == "system" - assert inputs[0]["content"] == TEST_SYSTEM_PROMPT - assert inputs[1]["role"] == "user" - assert inputs[1]["content"] == TEST_PROMPT - - -@pytest.mark.vcr -def test_litellm_completion_error(memory_logger): +@sync_async +async def test_litellm_completion_error(memory_logger, is_async): assert not memory_logger.pop() # Use a non-existent model to force an error fake_model = "non-existent-model" try: - litellm.completion(model=fake_model, messages=[{"role": "user", "content": TEST_PROMPT}]) - pytest.fail("Expected an exception but none was raised") - except Exception: - # We expect an error here - pass - - logs = memory_logger.pop() - assert len(logs) == 1 - log = logs[0] - assert log["project_id"] == PROJECT_NAME - # Check that we got a log entry with the fake model - assert fake_model in str(log) - - -@pytest.mark.vcr -@pytest.mark.asyncio -async def test_litellm_acompletion_error(memory_logger): - assert not memory_logger.pop() - - # Use a non-existent model to force an error - fake_model = "non-existent-model" - - try: - await litellm.acompletion(model=fake_model, messages=[{"role": "user", "content": TEST_PROMPT}]) + await _call( + is_async, + litellm.completion, + litellm.acompletion, + model=fake_model, + messages=[{"role": "user", "content": TEST_PROMPT}], + ) pytest.fail("Expected an exception but none was raised") except Exception: # We expect an error here @@ -775,22 +586,27 @@ def test_litellm_tool_calls(memory_logger): @pytest.mark.vcr -def test_litellm_responses_streaming_sync(memory_logger): +@pytest.mark.asyncio +@sync_async +async def test_litellm_responses_streaming(memory_logger, is_async): """Test the responses API with streaming.""" assert not memory_logger.pop() start = time.time() - stream = litellm.responses(model=TEST_MODEL, input="What's 12 + 12?", stream=True) - - chunks = [] - for chunk in stream: - if chunk.type == "response.output_text.delta": - chunks.append(chunk.delta) + stream = await _call( + is_async, + litellm.responses, + litellm.aresponses, + model=TEST_MODEL, + input="What's 12 + 12?", + stream=True, + ) + chunks = await _collect_stream(stream, is_async) end = time.time() - output = "".join(chunks) assert chunks assert len(chunks) > 1 + output = "".join(chunk.delta for chunk in chunks if getattr(chunk, "type", None) == "response.output_text.delta") assert "24" in output # Verify the span is created @@ -804,33 +620,6 @@ def test_litellm_responses_streaming_sync(memory_logger): assert "24" in str(span["output"]) -@pytest.mark.vcr -@pytest.mark.asyncio -async def test_litellm_aresponses_streaming_async(memory_logger): - """Test the async responses API with streaming.""" - assert not memory_logger.pop() - - start = time.time() - stream = await litellm.aresponses(model=TEST_MODEL, input="What's 12 + 12?", stream=True) - - chunks = [] - async for chunk in stream: - chunks.append(chunk) - end = time.time() - - assert chunks - assert len(chunks) > 1 - - # Verify the span is created - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - metrics = span["metrics"] - assert_metrics_are_valid(metrics, start, end) - assert span["metadata"]["stream"] is True - assert "What's 12 + 12?" in str(span["input"]) - - @pytest.mark.vcr @pytest.mark.asyncio async def test_litellm_async_streaming_with_break(memory_logger): @@ -947,11 +736,16 @@ def test_litellm_openrouter_no_booleans_in_metrics(memory_logger): @pytest.mark.vcr -def test_litellm_rerank(memory_logger): +@pytest.mark.asyncio +@sync_async +async def test_litellm_rerank(memory_logger, is_async): assert not memory_logger.pop() start = time.time() - response = litellm.rerank( + response = await _call( + is_async, + litellm.rerank, + litellm.arerank, model=RERANK_MODEL, query=RERANK_QUERY, documents=RERANK_DOCUMENTS, @@ -993,42 +787,6 @@ def test_litellm_rerank(memory_logger): assert metrics.get("search_units") == 1 -@pytest.mark.vcr -@pytest.mark.asyncio -async def test_litellm_arerank(memory_logger): - assert not memory_logger.pop() - - start = time.time() - response = await litellm.arerank( - model=RERANK_MODEL, - query=RERANK_QUERY, - documents=RERANK_DOCUMENTS, - top_n=2, - ) - end = time.time() - - assert response - assert response.results - assert len(response.results) == 2 - - spans = memory_logger.pop() - assert len(spans) == 1 - span = spans[0] - assert span["span_attributes"]["name"] == "Rerank" - assert span["metadata"]["provider"] == "cohere" - assert span["metadata"]["model"] == RERANK_MODEL - assert span["metadata"]["top_n"] == 2 - assert span["metadata"]["document_count"] == 3 - assert span["input"] == {"query": RERANK_QUERY, "documents": RERANK_DOCUMENTS} - assert isinstance(span["output"], list) - assert len(span["output"]) == 2 - - metrics = span["metrics"] - assert metrics["start"] >= start - assert metrics["end"] <= end - assert metrics.get("search_units") == 1 - - def test_litellm_parse_rerank_metrics_from_meta(): """Unit-level sanity check for ``_parse_rerank_metrics``. From 5ea964d39b38dfbf5ed99ad9c27ee37b9a5cfa2b Mon Sep 17 00:00:00 2001 From: Starfolk Date: Fri, 4 Sep 2026 20:02:59 +0000 Subject: [PATCH 2/3] test(litellm): simplify stream helper, restore cassette prompt parity Address /simplify review feedback: - _collect_stream now uses (async) comprehensions instead of a manual append loop. - test_litellm_image_generation sends the blue prompt on the async variant, matching the recorded [async] cassette body (the sync variant keeps red) so code and cassettes agree. Co-Authored-By: Claude Opus 4.8 --- .../braintrust/integrations/litellm/test_litellm.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/py/src/braintrust/integrations/litellm/test_litellm.py b/py/src/braintrust/integrations/litellm/test_litellm.py index 29a1ca21c..3e60932d6 100644 --- a/py/src/braintrust/integrations/litellm/test_litellm.py +++ b/py/src/braintrust/integrations/litellm/test_litellm.py @@ -48,14 +48,9 @@ async def _call(is_async, sync_fn, async_fn, *args, **kwargs): async def _collect_stream(stream, is_async): """Drain a sync or async LiteLLM stream into a list of chunks.""" - chunks = [] if is_async: - async for chunk in stream: - chunks.append(chunk) - else: - for chunk in stream: - chunks.append(chunk) - return chunks + return [chunk async for chunk in stream] + return list(stream) def _assert_speech_output_attachment(span) -> None: @@ -356,7 +351,8 @@ async def test_litellm_moderation(memory_logger, is_async): async def test_litellm_image_generation(memory_logger, is_async): assert not memory_logger.pop() - prompt = "A tiny red square on a white background" + # Distinct prompts per variant to match the recorded [sync]/[async] cassettes. + prompt = "A tiny blue square on a white background" if is_async else "A tiny red square on a white background" response = await _call( is_async, From 54e6e92c0a3c0269f6e1e2fac37abbc9dae2642b Mon Sep 17 00:00:00 2001 From: Starfolk Date: Fri, 4 Sep 2026 20:11:17 +0000 Subject: [PATCH 3/3] test(litellm): run sync variants outside the asyncio event loop Addresses PR review (P2): the parametrized tests were `async def` + @pytest.mark.asyncio, so the `is_async=False` variants invoked LiteLLM's sync entrypoints from inside a running event loop -- not how ordinary synchronous callers use them. LiteLLM and its transports can select loop-dependent code paths, so sync-only failures could be hidden even though both parameter values pass. The parametrized tests are now plain `def` functions. `_run` calls the sync entrypoint directly (no running loop) and drives the async entrypoint with `asyncio.run`; `_run_stream` does the same for streaming (await + async-drain under asyncio.run). The genuinely async-only tests (async_parallel_requests, async_streaming_with_break) keep @pytest.mark.asyncio. Validated: nox test_litellm(latest) and (1.74.0) both pass 38; pylint and pre-commit clean. Co-Authored-By: Claude Opus 4.8 --- .../integrations/litellm/test_litellm.py | 92 +++++++++---------- 1 file changed, 44 insertions(+), 48 deletions(-) diff --git a/py/src/braintrust/integrations/litellm/test_litellm.py b/py/src/braintrust/integrations/litellm/test_litellm.py index 3e60932d6..12ce1afea 100644 --- a/py/src/braintrust/integrations/litellm/test_litellm.py +++ b/py/src/braintrust/integrations/litellm/test_litellm.py @@ -36,21 +36,32 @@ # async LiteLLM entrypoint (e.g. ``acompletion``) over the sync one; the two # variants share a single test body and record to ``...[sync]`` / ``...[async]`` # cassettes. +# +# The parametrized tests are plain (non-``async``) functions so the ``sync`` +# variant calls LiteLLM with no running event loop, exactly as ordinary +# synchronous callers do -- ``_run``/``_run_stream`` drive the ``async`` variant +# with ``asyncio.run`` instead. Running the sync entrypoints inside a coroutine +# would let LiteLLM take loop-dependent code paths and hide sync-only failures. sync_async = pytest.mark.parametrize("is_async", [False, True], ids=["sync", "async"]) -async def _call(is_async, sync_fn, async_fn, *args, **kwargs): - """Invoke ``async_fn`` (awaited) when ``is_async`` else ``sync_fn``.""" +def _run(is_async, sync_fn, async_fn, *args, **kwargs): + """Call ``sync_fn`` directly, or drive ``async_fn`` with ``asyncio.run``.""" if is_async: - return await async_fn(*args, **kwargs) + return asyncio.run(async_fn(*args, **kwargs)) return sync_fn(*args, **kwargs) -async def _collect_stream(stream, is_async): - """Drain a sync or async LiteLLM stream into a list of chunks.""" +def _run_stream(is_async, sync_fn, async_fn, *args, **kwargs): + """Open a streaming request and drain it into a list of chunks.""" if is_async: - return [chunk async for chunk in stream] - return list(stream) + + async def _drain(): + stream = await async_fn(*args, **kwargs) + return [chunk async for chunk in stream] + + return asyncio.run(_drain()) + return list(sync_fn(*args, **kwargs)) def _assert_speech_output_attachment(span) -> None: @@ -75,13 +86,12 @@ def memory_logger(): @pytest.mark.vcr -@pytest.mark.asyncio @sync_async -async def test_litellm_completion_metrics(memory_logger, is_async) -> None: +def test_litellm_completion_metrics(memory_logger, is_async) -> None: assert not memory_logger.pop() start = time.time() - response = await _call( + response = _run( is_async, litellm.completion, litellm.acompletion, @@ -180,13 +190,12 @@ def test_litellm_prompt_caching_metrics(memory_logger) -> None: @pytest.mark.vcr -@pytest.mark.asyncio @sync_async -async def test_litellm_text_completion_metrics(memory_logger, is_async) -> None: +def test_litellm_text_completion_metrics(memory_logger, is_async) -> None: assert not memory_logger.pop() start = time.time() - response = await _call( + response = _run( is_async, litellm.text_completion, litellm.atext_completion, @@ -212,13 +221,12 @@ async def test_litellm_text_completion_metrics(memory_logger, is_async) -> None: @pytest.mark.vcr -@pytest.mark.asyncio @sync_async -async def test_litellm_completion_streaming(memory_logger, is_async): +def test_litellm_completion_streaming(memory_logger, is_async): assert not memory_logger.pop() start = time.time() - stream = await _call( + chunks = _run_stream( is_async, litellm.completion, litellm.acompletion, @@ -226,7 +234,6 @@ async def test_litellm_completion_streaming(memory_logger, is_async): messages=[{"role": "user", "content": TEST_PROMPT}], stream=True, ) - chunks = await _collect_stream(stream, is_async) end = time.time() # Verify streaming works @@ -256,13 +263,12 @@ async def test_litellm_completion_streaming(memory_logger, is_async): @pytest.mark.vcr -@pytest.mark.asyncio @sync_async -async def test_litellm_responses_metrics(memory_logger, is_async): +def test_litellm_responses_metrics(memory_logger, is_async): assert not memory_logger.pop() start = time.time() - response = await _call( + response = _run( is_async, litellm.responses, litellm.aresponses, @@ -291,12 +297,11 @@ async def test_litellm_responses_metrics(memory_logger, is_async): @pytest.mark.vcr -@pytest.mark.asyncio @sync_async -async def test_litellm_embedding(memory_logger, is_async): +def test_litellm_embedding(memory_logger, is_async): assert not memory_logger.pop() - response = await _call( + response = _run( is_async, litellm.embedding, litellm.aembedding, @@ -319,12 +324,11 @@ async def test_litellm_embedding(memory_logger, is_async): @pytest.mark.vcr -@pytest.mark.asyncio @sync_async -async def test_litellm_moderation(memory_logger, is_async): +def test_litellm_moderation(memory_logger, is_async): assert not memory_logger.pop() - response = await _call( + response = _run( is_async, litellm.moderation, litellm.amoderation, @@ -346,15 +350,14 @@ async def test_litellm_moderation(memory_logger, is_async): @pytest.mark.vcr -@pytest.mark.asyncio @sync_async -async def test_litellm_image_generation(memory_logger, is_async): +def test_litellm_image_generation(memory_logger, is_async): assert not memory_logger.pop() # Distinct prompts per variant to match the recorded [sync]/[async] cassettes. prompt = "A tiny blue square on a white background" if is_async else "A tiny red square on a white background" - response = await _call( + response = _run( is_async, litellm.image_generation, litellm.aimage_generation, @@ -378,12 +381,11 @@ async def test_litellm_image_generation(memory_logger, is_async): @pytest.mark.vcr -@pytest.mark.asyncio @sync_async -async def test_litellm_completion_with_system_prompt(memory_logger, is_async): +def test_litellm_completion_with_system_prompt(memory_logger, is_async): assert not memory_logger.pop() - response = await _call( + response = _run( is_async, litellm.completion, litellm.acompletion, @@ -407,13 +409,12 @@ async def test_litellm_completion_with_system_prompt(memory_logger, is_async): @pytest.mark.vcr -@pytest.mark.asyncio @sync_async -async def test_litellm_transcription(memory_logger, is_async): +def test_litellm_transcription(memory_logger, is_async): assert not memory_logger.pop() with open(TEST_AUDIO_FILE, "rb") as f: - response = await _call(is_async, litellm.transcription, litellm.atranscription, model="whisper-1", file=f) + response = _run(is_async, litellm.transcription, litellm.atranscription, model="whisper-1", file=f) assert response assert response.text == "you" @@ -430,12 +431,11 @@ async def test_litellm_transcription(memory_logger, is_async): @pytest.mark.vcr -@pytest.mark.asyncio @sync_async -async def test_litellm_speech(memory_logger, is_async): +def test_litellm_speech(memory_logger, is_async): assert not memory_logger.pop() - response = await _call( + response = _run( is_async, litellm.speech, litellm.aspeech, @@ -460,16 +460,15 @@ async def test_litellm_speech(memory_logger, is_async): @pytest.mark.vcr -@pytest.mark.asyncio @sync_async -async def test_litellm_completion_error(memory_logger, is_async): +def test_litellm_completion_error(memory_logger, is_async): assert not memory_logger.pop() # Use a non-existent model to force an error fake_model = "non-existent-model" try: - await _call( + _run( is_async, litellm.completion, litellm.acompletion, @@ -582,14 +581,13 @@ def test_litellm_tool_calls(memory_logger): @pytest.mark.vcr -@pytest.mark.asyncio @sync_async -async def test_litellm_responses_streaming(memory_logger, is_async): +def test_litellm_responses_streaming(memory_logger, is_async): """Test the responses API with streaming.""" assert not memory_logger.pop() start = time.time() - stream = await _call( + chunks = _run_stream( is_async, litellm.responses, litellm.aresponses, @@ -597,7 +595,6 @@ async def test_litellm_responses_streaming(memory_logger, is_async): input="What's 12 + 12?", stream=True, ) - chunks = await _collect_stream(stream, is_async) end = time.time() assert chunks @@ -732,13 +729,12 @@ def test_litellm_openrouter_no_booleans_in_metrics(memory_logger): @pytest.mark.vcr -@pytest.mark.asyncio @sync_async -async def test_litellm_rerank(memory_logger, is_async): +def test_litellm_rerank(memory_logger, is_async): assert not memory_logger.pop() start = time.time() - response = await _call( + response = _run( is_async, litellm.rerank, litellm.arerank,