diff --git a/aws_lambda_builders/workflows/python_uv/DESIGN.md b/aws_lambda_builders/workflows/python_uv/DESIGN.md index 61a27c092..ac1b0d293 100644 --- a/aws_lambda_builders/workflows/python_uv/DESIGN.md +++ b/aws_lambda_builders/workflows/python_uv/DESIGN.md @@ -205,7 +205,7 @@ config = { "no_cache": False, # Disable caching "prerelease": "disallow", # Handle pre-release versions "resolution": "highest", # Resolution strategy - "compile_bytecode": True, # Compile .pyc files + "compile_bytecode": True, # Compile unchecked-hash .pyc with the target Python when enabled "exclude_newer": None, # Exclude packages newer than date "generate_hashes": False, # Generate package hashes } diff --git a/aws_lambda_builders/workflows/python_uv/packager.py b/aws_lambda_builders/workflows/python_uv/packager.py index b6c88a9d7..19285e9c0 100644 --- a/aws_lambda_builders/workflows/python_uv/packager.py +++ b/aws_lambda_builders/workflows/python_uv/packager.py @@ -54,6 +54,17 @@ def get_uv_version(self) -> Optional[str]: pass return None + def find_python(self, python_version: str) -> Optional[str]: + """Find an already-installed interpreter matching the requested Python version.""" + rc, stdout, stderr = self.run_uv_command( + ["python", "find", "--no-project", "--no-python-downloads", python_version] + ) + if rc == 0 and stdout: + return stdout.strip() + diagnostic = (stderr or "").strip() or (stdout or "").strip() or "no diagnostic output" + LOG.warning("Could not locate target Python %s via uv (exit code %d): %s", python_version, rc, diagnostic) + return None + def run_uv_command(self, args: List[str], cwd: Optional[str] = None, env: Optional[Dict[str, str]] = None) -> tuple: """ Execute UV command with given arguments. @@ -133,15 +144,33 @@ def install_requirements( # Resolve --target to an absolute path: UV runs with cwd set to the project directory, so a # relative target (e.g. the incremental-build dependencies dir) would otherwise be created # under the source directory instead of the build root. - args.extend(["--target", os.path.abspath(target_dir)]) + target_dir = os.path.abspath(target_dir) + args.extend(["--target", target_dir]) - # Add configuration arguments + target_python = None + if config.compile_bytecode: + if python_version: + target_python = self._uv.find_python(python_version) + else: + LOG.warning("Target Python version is unavailable; skipping bytecode compilation") + + # Add configuration arguments that are independent of interpreter selection. args.extend(config.to_uv_args()) # Add platform-specific arguments if python_version: args.extend(["--python-version", python_version]) + # Use the exact interpreter found by UV instead of repeating a version request that + # could resolve differently when the install command runs. + if target_python: + args.extend(["--python", target_python]) + + # UV currently produces timestamp-based bytecode. SAM packages files with normalized + # timestamps, so that bytecode would be rejected by the Lambda runtime after deployment. + # Keep UV compilation disabled and generate unchecked-hash bytecode below instead. + args.append("--no-compile-bytecode") + if platform and architecture: # UV pip install uses --python-platform format # Map Lambda architectures to UV platform strings @@ -162,6 +191,34 @@ def install_requirements( LOG.debug("UV pip install completed successfully: %s", stdout) + if target_python: + self._compile_bytecode(target_python, target_dir) + + def _compile_bytecode(self, python_executable: str, target_dir: str) -> None: + """Compile reusable bytecode with the interpreter selected for the target runtime.""" + command = [ + python_executable, + "-m", + "compileall", + "-f", + "-q", + "--invalidation-mode", + "unchecked-hash", + target_dir, + ] + LOG.debug("Compiling unchecked-hash bytecode: %s", " ".join(command)) + rc, stdout, stderr = self._osutils.run_subprocess(command) + if rc != 0: + diagnostic = stderr.strip() or stdout.strip() or "no diagnostic output" + LOG.warning( + "Could not compile unchecked-hash bytecode with target Python (exit code %d): %s", + rc, + diagnostic, + ) + return + + LOG.debug("Python bytecode compilation completed successfully: %s", stdout) + class PythonUvDependencyBuilder: """High-level dependency builder that orchestrates UV operations.""" diff --git a/aws_lambda_builders/workflows/python_uv/utils.py b/aws_lambda_builders/workflows/python_uv/utils.py index cb7bfeb5b..220291882 100644 --- a/aws_lambda_builders/workflows/python_uv/utils.py +++ b/aws_lambda_builders/workflows/python_uv/utils.py @@ -25,7 +25,16 @@ def run_subprocess(self, cmd, cwd=None, env=None): env = self.original_environ() try: - result = subprocess.run(cmd, cwd=cwd, env=env, capture_output=True, text=True, check=False) + result = subprocess.run( + cmd, + cwd=cwd, + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) return result.returncode, result.stdout, result.stderr except Exception as e: return 1, "", str(e) diff --git a/tests/unit/workflows/python_uv/test_packager.py b/tests/unit/workflows/python_uv/test_packager.py index 1c199ae24..2bdf1ed63 100644 --- a/tests/unit/workflows/python_uv/test_packager.py +++ b/tests/unit/workflows/python_uv/test_packager.py @@ -91,11 +91,57 @@ def test_get_uv_version_failure(self, mock_osutils_class): self.assertIsNone(version) + @patch("aws_lambda_builders.workflows.python_uv.packager.OSUtils") + def test_find_python_success(self, mock_osutils_class): + mock_osutils = Mock() + mock_osutils.which.return_value = "/usr/bin/uv" + mock_osutils.run_subprocess.return_value = (0, "/usr/bin/python3.13\n", "") + mock_osutils_class.return_value = mock_osutils + + subprocess_uv = SubprocessUv() + + self.assertEqual(subprocess_uv.find_python("3.13"), "/usr/bin/python3.13") + mock_osutils.run_subprocess.assert_called_once_with( + ["/usr/bin/uv", "python", "find", "--no-project", "--no-python-downloads", "3.13"], + cwd=None, + env=None, + ) + + @patch("aws_lambda_builders.workflows.python_uv.packager.OSUtils") + def test_find_python_failure(self, mock_osutils_class): + mock_osutils = Mock() + mock_osutils.which.return_value = "/usr/bin/uv" + mock_osutils.run_subprocess.return_value = (2, "", "interpreter not found") + mock_osutils_class.return_value = mock_osutils + + subprocess_uv = SubprocessUv() + + with self.assertLogs("aws_lambda_builders.workflows.python_uv.packager", level="WARNING") as logs: + self.assertIsNone(subprocess_uv.find_python("3.9")) + + self.assertIn("Could not locate target Python 3.9 via uv (exit code 2)", logs.output[0]) + self.assertIn("interpreter not found", logs.output[0]) + + @patch("aws_lambda_builders.workflows.python_uv.packager.OSUtils") + def test_find_python_failure_without_diagnostics(self, mock_osutils_class): + mock_osutils = Mock() + mock_osutils.which.return_value = "/usr/bin/uv" + mock_osutils.run_subprocess.return_value = (1, None, None) + mock_osutils_class.return_value = mock_osutils + + subprocess_uv = SubprocessUv() + + with self.assertLogs("aws_lambda_builders.workflows.python_uv.packager", level="WARNING") as logs: + self.assertIsNone(subprocess_uv.find_python("3.9")) + + self.assertIn("no diagnostic output", logs.output[0]) + class TestUvRunner(TestCase): def setUp(self): self.mock_subprocess_uv = Mock() self.mock_subprocess_uv.uv_executable = "/usr/bin/uv" + self.mock_subprocess_uv.find_python.return_value = None self.mock_osutils = Mock() self.uv_runner = UvRunner(uv_subprocess=self.mock_subprocess_uv, osutils=self.mock_osutils) @@ -123,6 +169,108 @@ def test_install_requirements_success(self): self.assertIn("install", args_called) self.assertIn("-r", args_called) self.assertIn("/path/to/requirements.txt", args_called) + self.mock_subprocess_uv.find_python.assert_called_once_with("3.9") + + def test_install_requirements_compiles_unchecked_hash_bytecode_by_default(self): + self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "") + self.mock_subprocess_uv.find_python.return_value = "/usr/bin/python3.13" + self.mock_osutils.run_subprocess.return_value = (0, "", "") + + self.uv_runner.install_requirements( + requirements_path="/path/to/requirements.txt", + target_dir="/target", + python_version="3.13", + ) + + args_called = self.mock_subprocess_uv.run_uv_command.call_args[0][0] + self.assertIn("--no-compile-bytecode", args_called) + self.assertNotIn("--compile-bytecode", args_called) + self.assertEqual(args_called[args_called.index("--python") + 1], "/usr/bin/python3.13") + self.assertEqual(args_called[args_called.index("--python-version") + 1], "3.13") + self.mock_subprocess_uv.find_python.assert_called_once_with("3.13") + self.mock_osutils.run_subprocess.assert_called_once_with( + [ + "/usr/bin/python3.13", + "-m", + "compileall", + "-f", + "-q", + "--invalidation-mode", + "unchecked-hash", + os.path.abspath("/target"), + ] + ) + + def test_install_requirements_skips_bytecode_when_target_python_is_unavailable(self): + self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "") + self.mock_subprocess_uv.find_python.return_value = None + + self.uv_runner.install_requirements( + requirements_path="/path/to/requirements.txt", + target_dir="/target", + config=UvConfig(compile_bytecode=True), + python_version="3.9", + ) + + args_called = self.mock_subprocess_uv.run_uv_command.call_args[0][0] + self.assertIn("--no-compile-bytecode", args_called) + self.assertNotIn("--compile-bytecode", args_called) + self.assertNotIn("--python", args_called) + self.assertEqual(args_called[args_called.index("--python-version") + 1], "3.9") + self.mock_subprocess_uv.find_python.assert_called_once_with("3.9") + self.mock_osutils.run_subprocess.assert_not_called() + + def test_install_requirements_skips_bytecode_without_target_python_version(self): + self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "") + + with self.assertLogs("aws_lambda_builders.workflows.python_uv.packager", level="WARNING") as logs: + self.uv_runner.install_requirements( + requirements_path="/path/to/requirements.txt", + target_dir="/target", + config=UvConfig(compile_bytecode=True), + ) + + args_called = self.mock_subprocess_uv.run_uv_command.call_args[0][0] + self.assertIn("--no-compile-bytecode", args_called) + self.assertNotIn("--compile-bytecode", args_called) + self.assertNotIn("--python", args_called) + self.mock_subprocess_uv.find_python.assert_not_called() + self.assertIn("Target Python version is unavailable", logs.output[0]) + self.mock_osutils.run_subprocess.assert_not_called() + + def test_install_requirements_does_not_select_python_when_bytecode_disabled(self): + self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "") + + self.uv_runner.install_requirements( + requirements_path="/path/to/requirements.txt", + target_dir="/target", + config=UvConfig(compile_bytecode=False), + python_version="3.13", + ) + + args_called = self.mock_subprocess_uv.run_uv_command.call_args[0][0] + self.assertIn("--no-compile-bytecode", args_called) + self.assertNotIn("--compile-bytecode", args_called) + self.assertNotIn("--python", args_called) + self.assertEqual(args_called[args_called.index("--python-version") + 1], "3.13") + self.mock_subprocess_uv.find_python.assert_not_called() + self.mock_osutils.run_subprocess.assert_not_called() + + def test_install_requirements_keeps_build_non_fatal_when_bytecode_compilation_fails(self): + self.mock_subprocess_uv.run_uv_command.return_value = (0, "success", "") + self.mock_subprocess_uv.find_python.return_value = "/usr/bin/python3.13" + self.mock_osutils.run_subprocess.return_value = (1, "", "compile error") + + with self.assertLogs("aws_lambda_builders.workflows.python_uv.packager", level="WARNING") as logs: + self.uv_runner.install_requirements( + requirements_path="/path/to/requirements.txt", + target_dir="/target", + config=UvConfig(compile_bytecode=True), + python_version="3.13", + ) + + self.assertIn("Could not compile unchecked-hash bytecode with target Python (exit code 1)", logs.output[0]) + self.assertIn("compile error", logs.output[0]) def test_install_requirements_resolves_relative_target_to_absolute(self): # UV runs with cwd=project_dir, so a relative --target must be resolved to an absolute path diff --git a/tests/unit/workflows/python_uv/test_utils.py b/tests/unit/workflows/python_uv/test_utils.py index f15e3e550..878a51d4d 100644 --- a/tests/unit/workflows/python_uv/test_utils.py +++ b/tests/unit/workflows/python_uv/test_utils.py @@ -1,7 +1,7 @@ import os import tempfile from unittest import TestCase -from unittest.mock import Mock +from unittest.mock import Mock, patch from aws_lambda_builders.workflows.python_uv.utils import ( OSUtils, @@ -35,6 +35,25 @@ def test_run_subprocess_failure(self): rc, stdout, stderr = self.osutils.run_subprocess(["false"]) self.assertEqual(rc, 1) + @patch("aws_lambda_builders.workflows.python_uv.utils.subprocess.run") + def test_run_subprocess_decodes_utf8_with_replacement(self, mock_run): + mock_run.return_value = Mock(returncode=0, stdout="success", stderr="") + env = {"UV_TEST": "1"} + + result = self.osutils.run_subprocess(["uv", "--version"], cwd="/work", env=env) + + self.assertEqual(result, (0, "success", "")) + mock_run.assert_called_once_with( + ["uv", "--version"], + cwd="/work", + env=env, + capture_output=True, + text=True, + encoding="utf-8", + errors="replace", + check=False, + ) + class TestDetectUvManifest(TestCase): def test_detect_uv_manifest_no_files(self): @@ -107,6 +126,7 @@ def test_uv_config_defaults(self): config = UvConfig() args = config.to_uv_args() self.assertEqual(args, []) + self.assertTrue(config.compile_bytecode) def test_uv_config_with_index_url(self): config = UvConfig(index_url="https://pypi.org/simple/")