Add support for @ENV as an exe preference

.github/workflows/tests.yml

@@ -53,12 +53,15 @@ jobs:
         with:
           python-version: ${{ matrix.pyversion }}
 
+      # Always build PyCall (and hence Conda) so that Conda's root env directory exists;
+      # otherwise Conda fails to precompile on cached runners, which can cause errors.
+      # Use a temporary environment because `--project=test` has no manifest on Julia
+      # versions without workspace support (< 1.12).
       - name: Build PyCall
-        if: ${{ matrix.pythonexe == 'python' }}
         run: |
-          julia --project=test -e 'import Pkg; Pkg.build("PyCall")'
+          julia -e 'import Pkg; Pkg.activate(temp=true); Pkg.add("PyCall"); Pkg.build("PyCall")'
         env:
-          PYTHON: ${{ steps.setup-python.outputs.python-path }}
+          PYTHON: ${{ case(matrix.pythonexe == 'python', steps.setup-python.outputs.python-path, 'python') }}
 
       - name: Run tests
         uses: julia-actions/julia-runtest@v1

docs/src/pythoncall.md

@@ -309,6 +309,15 @@ special values:
 - `@CondaPkg`: Use Python from CondaPkg (the default).
 - `@PyCall`: Use the same Python as PyCall. [See here](@ref faq-pycall).
 - `@venv`: Use Python from a `.venv` virtual environment in the current active project.
+- `@ENV`: This value should only be set as the `exe` preference. When set,
+  PythonCall will look up the `exe` value explicitly from the
+  `JULIA_PYTHONCALL_EXE` environment variable, which _must_ be a valid path or
+  command name (see below). The advantage of setting `@ENV` as a preference is
+  that CondaPkg.jl will not be loaded, which saves on load time, while you still
+  keep the ability to change Python environments based on environment
+  variables. This can be helpful when using shared Julia environments on a
+  cluster so that users can easily switch Python environments without modifying
+  the depot preferences.
 
 Otherwise, the value is interpreted as:
 - An absolute path to a Python executable.

pysrc/juliacall/__init__.py

@@ -256,7 +256,6 @@ def args_from_config(config):
 
     # override some environment variables
     # we do this here because PythonCall is initialised during jl_init if it is in a sysimg
-    os.environ['JULIA_PYTHONCALL_EXE'] = sys.executable or ''
     os.environ['__JULIA_PYTHONCALL_EMBEDDED_LIBPTR__'] = hex(c.pythonapi._handle)
 
     # initialise julia

pytest/test_all.py

@@ -5,6 +5,14 @@ def test_import():
     import juliacall
 
 
+def test_python_executable_path():
+    import sys
+    import juliacall
+
+    jl = juliacall.Main
+    assert str(jl.PythonCall.python_executable_path()) == sys.executable
+
+
 def test_newmodule():
     import juliacall
 

src/C/context.jl

@@ -118,11 +118,20 @@ function init_context()
         Py_IsInitialized() == 0 && error("Python is not already initialized.")
         CTX.is_initialized = true
         CTX.which = :embedded
-        exe_path = Utils.getpref_exe()
-        if exe_path != ""
-            CTX.exe_path = exe_path
-            # this ensures PyCall uses the same Python interpreter
-            get!(ENV, "PYTHON", exe_path)
+        # The running interpreter is the source of truth, so ignore the exe preference
+        exe_ptr = PySys_GetObject("executable")
+        if exe_ptr != C_NULL
+            str_ptr = PyUnicode_AsUTF8AndSize(exe_ptr, C_NULL)
+            if str_ptr == C_NULL
+                PyErr_Clear()
+            else
+                exe_path = Base.unsafe_string(str_ptr)
+                if exe_path != ""
+                    CTX.exe_path = exe_path
+                    # this ensures PyCall uses the same Python interpreter
+                    get!(ENV, "PYTHON", exe_path)
+                end
+            end
         end
     else
         # Find Python executable
@@ -160,9 +169,16 @@ function init_context()
             else
                 exe_path = abspath(exe_path, "bin", "python")::String
             end
-        elseif startswith(exe_path, "@")
+        elseif startswith(exe_path, "@") && exe_path != "@ENV"
             error("invalid exe: $exe_path")
         else
+            if exe_path == "@ENV"
+                exe_path = get(ENV, "JULIA_PYTHONCALL_EXE", "")
+                if isempty(exe_path)
+                    error("PythonCall's `exe` preference is set to `@ENV`, but JULIA_PYTHONCALL_EXE is not set.")
+                end
+            end
+
             # Otherwise we use the Python specified
             CTX.which = :unknown
             if isabspath(exe_path)

src/C/pointers.jl

@@ -28,6 +28,8 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}(
     :PyImport_Import => (PyPtr,) => PyPtr,
     :PyImport_ImportModuleLevelObject => (PyPtr, PyPtr, PyPtr, PyPtr, Cint) => PyPtr,
     :PyImport_GetModuleDict => () => PyPtr, # borrowed
+    # SYS
+    :PySys_GetObject => (Ptr{Cchar},) => PyPtr, # borrowed
     # MODULE
     :PyModule_GetDict => (PyPtr,) => PyPtr, # borrowed
     # ERRORS
@@ -142,6 +144,7 @@ const CAPI_FUNC_SIGS = Dict{Symbol,Pair{Tuple,Type}}(
     # STR
     :PyUnicode_DecodeUTF8 => (Ptr{Cchar}, Py_ssize_t, Ptr{Cchar}) => PyPtr,
     :PyUnicode_AsUTF8String => (PyPtr,) => PyPtr,
+    :PyUnicode_AsUTF8AndSize => (PyPtr, Ptr{Py_ssize_t}) => Ptr{Cchar}, # borrowed
     :PyUnicode_InternInPlace => (Ptr{PyPtr},) => Cvoid,
     # BYTES
     :PyBytes_FromStringAndSize => (Ptr{Cchar}, Py_ssize_t) => PyPtr,

test/C.jl

@@ -16,3 +16,31 @@
         @test PythonCall.python_version().major == 3
     end
 end
+
+@testitem "exe = @ENV" begin
+    mktempdir() do dir
+        # A throwaway environment prepended to the load path overrides the `exe`
+        # preference for a child process while still resolving PythonCall from ours.
+        write(
+            joinpath(dir, "Project.toml"),
+            "[extras]\nPythonCall = \"6099a3de-0909-46bc-b1f4-468b9a2dfc0d\"\n",
+        )
+        write(joinpath(dir, "LocalPreferences.toml"), "[PythonCall]\nexe = \"@ENV\"\n")
+        sep = Sys.iswindows() ? ";" : ":"
+        loadpath = join([dir, Base.active_project(), "@stdlib"], sep)
+        exe = PythonCall.python_executable_path()
+        code = "using PythonCall; print(PythonCall.python_executable_path())"
+        cmd = `$(Base.julia_cmd()) --startup-file=no -e $code`
+
+        # variable set: it is used as the executable
+        withset = addenv(cmd, "JULIA_LOAD_PATH" => loadpath, "JULIA_PYTHONCALL_EXE" => exe)
+        @test readchomp(withset) == exe
+
+        # variable unset: fails at load time with a clear message
+        unset = addenv(cmd, "JULIA_LOAD_PATH" => loadpath, "JULIA_PYTHONCALL_EXE" => nothing)
+        err = IOBuffer()
+        proc = run(pipeline(ignorestatus(unset), stdout = devnull, stderr = err))
+        @test !success(proc)
+        @test occursin("JULIA_PYTHONCALL_EXE is not set", String(take!(err)))
+    end
+end