Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -25,11 +25,18 @@ nox -s system -- tests/system/_async

## 3. Verify Sync/Async Parity
Run the cross-sync generation tool and ensure no regressions in the synchronous codebase.

`generate.py` must be pointed at a **directory**; it only rewrites files reachable from
that directory that carry a `__CROSS_SYNC_OUTPUT__` annotation.
```bash
python3 .cross_sync/generate.py
PYTHONPATH=.cross_sync python3 .cross_sync/generate.py google/cloud/spanner_v1/_async/
git diff --exit-code google/cloud/spanner_v1/
nox -s unit-3.14
nox -s system-3.14
```
A non-empty `git diff` here means the generated sync code has drifted from
`google/cloud/spanner_v1/_async/`. Fix it in the `_async/` source, never in the
generated artifact.

## 4. Check for Coroutine Leaks
Ensure all asynchronous GAPIC calls are properly awaited. Search for any unawaited coroutines in the `_async` directory.
Expand Down
85 changes: 72 additions & 13 deletions packages/google-cloud-spanner/.cross_sync/generate.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,56 @@ def extract_header_comments(file_path) -> str:
return "".join(header)


# Keep these in sync with the `format` / `lint` sessions in noxfile.py, otherwise
# regenerating the artifacts will produce a spurious formatting-only diff.
RUFF_TARGET_VERSION = "py310"
RUFF_LINE_LENGTH = "88"


def format_with_ruff(source: str, filename: str) -> str:
"""
Format generated source with ruff, the formatter used by this repository.

Runs two passes over stdin, mirroring `nox -s format`:
1. `ruff check --select I,F401 --fix` to sort imports and drop the
imports that became unused during the async -> sync conversion.
2. `ruff format` to apply the code style.

Args:
source: the generated python source
filename: the path the source will be written to. Only used to give
ruff a sensible filename for diagnostics.
Returns:
the formatted source
"""
import shutil
import subprocess
import sys

ruff = shutil.which("ruff")
base_command = [ruff] if ruff else [sys.executable, "-m", "ruff"]
shared_args = [
f"--target-version={RUFF_TARGET_VERSION}",
"--line-length",
RUFF_LINE_LENGTH,
"--stdin-filename",
filename,
"-",
]
passes = [
base_command + ["check", "--select", "I,F401", "--fix", "--quiet", *shared_args],
base_command + ["format", *shared_args],
]
for command in passes:
result = subprocess.run(command, input=source, capture_output=True, text=True)
if result.returncode != 0:
raise RuntimeError(
f"ruff failed for {filename}: {' '.join(command)}\n{result.stderr}"
)
source = result.stdout
return source


class CrossSyncOutputFile:

def __init__(self, output_path: str, ast_tree, header: str | None = None):
Expand All @@ -51,18 +101,12 @@ def render(self, with_formatter=True, save_to_disk: bool = True) -> str:
Render the file to a string, and optionally save to disk

Args:
with_formatter: whether to run the output through black before returning
with_formatter: whether to run the output through ruff before returning
save_to_disk: whether to write the output to the file path
"""
full_str = self.header + ast.unparse(self.tree)
if with_formatter:
import black # type: ignore
import autoflake # type: ignore

full_str = black.format_str(
autoflake.fix_code(full_str, remove_all_unused_imports=True),
mode=black.FileMode(),
)
full_str = format_with_ruff(full_str, self.output_path)
if save_to_disk:
import os
os.makedirs(os.path.dirname(self.output_path), exist_ok=True)
Expand All @@ -71,12 +115,18 @@ def render(self, with_formatter=True, save_to_disk: bool = True) -> str:
return full_str


def convert_files_in_dir(directory: str) -> set[CrossSyncOutputFile]:
def convert_path(search_path: str) -> set[CrossSyncOutputFile]:
import glob
from transformers import CrossSyncFileProcessor

# find all python files in the directory
files = glob.glob(directory + "/**/*.py", recursive=True)
if os.path.isfile(search_path):
files = [search_path]
elif os.path.isdir(search_path):
files = glob.glob(search_path + "/**/*.py", recursive=True)
else:
print(f"Path does not exist: {search_path}")
sys.exit(1)

# keep track of the output files pointed to by the annotated classes
artifacts: set[CrossSyncOutputFile] = set()
file_transformer = CrossSyncFileProcessor()
Expand All @@ -100,13 +150,22 @@ def save_artifacts(artifacts: Sequence[CrossSyncOutputFile]):


if __name__ == "__main__":
import os
import sys

if len(sys.argv) < 2:
print("Usage: python .cross_sync/generate.py <directory>")
print("Usage: python .cross_sync/generate.py <directory_or_file>")
sys.exit(1)

search_root = sys.argv[1]
outputs = convert_files_in_dir(search_root)
if not os.path.exists(search_root):
print(f"Path does not exist: {search_root}")
sys.exit(1)

outputs = convert_path(search_root)
if not outputs:
print(f"No __CROSS_SYNC_OUTPUT__ annotated files found under {search_root}")
sys.exit(1)

print(f"Generated {len(outputs)} artifacts: {[a.output_path for a in outputs]}")
save_artifacts(outputs)
Loading
Loading