Merge branch 'main' into v1

Author: Christopher Doris

.github/workflows/docscleanup.yml

@@ -15,16 +15,15 @@ jobs:
 
       - name: Delete preview and history
         run: |
-            git config user.name "Documenter.jl"
-            git config user.email "[email protected]"
-            git rm -rf "previews/PR$PRNUM"
-            git commit -m "delete preview"
-            git branch gh-pages-new $(echo "delete history" | git commit-tree HEAD^{tree})
+            if [ -d "$DIR" ]; then
+              git config user.name "Documenter.jl"
+              git config user.email "[email protected]"
+              git rm -rf "$DIR"
+              git commit -m "delete preview"
+              git branch gh-pages-new $(echo "delete history" | git commit-tree HEAD^{tree})
+              git push --force origin gh-pages-new:gh-pages
+            fi
         env:
-            PRNUM: ${{ github.event.number }}
+            DIR: "previews/PR${{ github.event.number }}"
 
-      - name: Push changes
-        run: |
-            git push --force origin gh-pages-new:gh-pages
-            
 # Workflow copied from https://github.com/CliMA/TimeMachine.jl

Project.toml

@@ -17,10 +17,19 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 UnsafePointers = "e17b2a0c-0bdf-430a-bd0c-3a23cae4ff39"
 
 [compat]
+Aqua = "0 - 999"
 CondaPkg = "0.2.21"
+Dates = "1"
+Libdl = "1"
 MacroTools = "0.5"
+Markdown = "1"
+Pkg = "1"
+REPL = "1"
 Requires = "1"
+Serialization = "1"
 Tables = "1"
+Test = "1"
+TestItemRunner = "0 - 999"
 UnsafePointers = "1"
 julia = "1.6.1"
 

README.md

@@ -1,4 +1,4 @@
-<h1><img src="https://raw.githubusercontent.com/JuliaPy/PythonCall.jl/main/docs/src/assets/logo.png" alt="PythonCall.jl logo" style="width: 100px;"><br>PythonCall &amp;&nbsp;JuliaCall</h1>
+<h1><img src="docs/src/assets/logo.png" alt="PythonCall.jl logo" style="width: 100px;"><br>PythonCall &amp;&nbsp;JuliaCall</h1>
 
 [![Project Status: Active – The project has reached a stable, usable state and is being actively developed.](https://www.repostatus.org/badges/latest/active.svg)](https://www.repostatus.org/#active)
 [![Stable Documentation](https://img.shields.io/badge/docs-stable-blue.svg)](https://juliapy.github.io/PythonCall.jl/stable)

docs/make.jl

@@ -5,6 +5,9 @@ include("customdocs.jl")
 makedocs(
     sitename = "PythonCall & JuliaCall",
     modules = [PythonCall],
+    format = Documenter.HTML(
+        assets = ["assets/favicon.ico"],
+    ),
     warnonly = [:missing_docs], # avoid raising error when docs are missing
     pages = [
         "Home" => "index.md",

docs/src/assets/favicon.ico

@@ -18,17 +18,6 @@ Some rules if you are writing multithreaded code:
 
 Related issues: [#201](https://github.com/JuliaPy/PythonCall.jl/issues/201), [#202](https://github.com/JuliaPy/PythonCall.jl/issues/202)
 
-## Does it work on Apple silicon (ARM, M1, M2, ...)?
-
-Maybe. Your mileage may vary.
-
-In general, PythonCall and JuliaCall are only supported on platforms with
-[Tier 1](https://julialang.org/downloads/#supported_platforms) level of support by Julia.
-Currently, Apple silicon is Tier 2, so is not supported.
-
-Due to time constraints, issues affecting only unsupported platforms will not be
-investigated. It is much more likely to be an issue with Julia itself than PythonCall.
-
 ## Issues when Numpy arrays are expected
 
 When a Julia array is passed to Python, it is wrapped as a [`ArrayValue`](#juliacall.ArrayValue).
@@ -92,4 +81,6 @@ Related issues: [#255](https://github.com/JuliaPy/PythonCall.jl/issues/255)
 
 ## Can I use JuliaCall to run Julia inside applications with embedded Python?
 
-Yes, it may be possible. See an example of how to have Julia running inside the Python that is running inside Blender here https://discourse.julialang.org/t/running-julia-inside-blender-through-vscode-using-pythoncall-juliacall/96838.
+Yes, it may be possible. A good example of that is having Julia running inside the Python that is running inside Blender, as presented in [this Discourse post](https://discourse.julialang.org/t/running-julia-inside-blender-through-vscode-using-pythoncall-juliacall/96838/6).
+From the point that one has JuliaCall running inside Python, if it has access to the terminal, one can even launch a Julia REPL there, and if needed connect with VSCode Julia extension to it.
+The full Python script to install, launch JuliaCall, and launch a Julia REPL in Blender is [here](https://gist.github.com/cdsousa/d820d27174238c0d48e5252355584172).

docs/src/assets/logo.png

@@ -9,7 +9,7 @@ pip install juliacall
 
 Developers may wish to clone the repo (https://github.com/JuliaPy/PythonCall.jl) directly
 and pip install the module in editable mode. You should add `"dev":true, "path":"../.."` to
-`python/juliacall/juliapkg.json` to ensure you use the development version of PythonCall
+`pysrc/juliacall/juliapkg.json` to ensure you use the development version of PythonCall
 in conjunction with JuliaCall.
 
 ## Getting started
@@ -119,7 +119,7 @@ be configured in two ways:
 | `-X juliacall-min-optlevel=<0\|1\|2\|3>` | `PYTHON_JULIACALL_MIN_OPTLEVEL=<0\|1\|2\|3>` | Optimization level. |
 | `-X juliacall-optimize=<0\|1\|2\|3>` | `PYTHON_JULIACALL_OPTIMIZE=<0\|1\|2\|3>` | Minimum optimization level. |
 | `-X juliacall-procs=<N\|auto>` | `PYTHON_JULIACALL_PROCS=<N\|auto>` | Launch N local worker process. |
-| `-X juliacall-startup-file=<yes|no>` | `PYTHON_JULIACALL_STARTUP_FILE=<yes|no>` | Enable or disable your startup.jl file. |
+| `-X juliacall-startup-file=<yes\|no>` | `PYTHON_JULIACALL_STARTUP_FILE=<yes\|no>` | Enable or disable your startup.jl file. |
 | `-X juliacall-sysimage=<file>` | `PYTHON_JULIACALL_SYSIMAGE=<file>` | Use the given system image. |
 | `-X juliacall-threads=<N\|auto>` | `PYTHON_JULIACALL_THREADS=<N\|auto>` | Launch N threads. |
 | `-X juliacall-warn-overwrite=<yes\|no>` | `PYTHON_JULIACALL_WARN_OVERWRITE=<yes\|no>` | Enable or disable method overwrite warnings. |

docs/src/faq.md

@@ -6,6 +6,7 @@
 ## Unreleased
 * New unexported functions: `python_executable_path`, `python_library_path`, `python_library_handle` and `python_version`.
 * `Py` is now treated as a scalar when broadcasting.
+* `PyArray` is now serializable.
 * Bug fixes.
 
 ## 0.9.15 (2023-10-25)

docs/src/juliacall.md

@@ -109,11 +109,16 @@ def int_option(name, *, accept_auto=False, **kw):
             raise ValueError(f'{s}: expecting an int'+(' or auto' if accept_auto else ""))
 
     def args_from_config():
-        argv = ["--"+opt[4:].replace("_", "-")+"="+val for opt, val in CONFIG.items()
-                if val is not None and opt.startswith("opt_")]
-        argv = [CONFIG['exepath']]+argv
-        if sys.version_info[0] >= 3:
-            argv = [s.encode("utf-8") for s in argv]
+        argv = [CONFIG['exepath']]
+        for opt, val in CONFIG.items():
+            if opt.startswith('opt_'):
+                if val is None:
+                    if opt == 'opt_handle_signals':
+                        val = 'no'
+                    else:
+                        continue
+                argv.append('--' + opt[4:].replace('_', '-') + '=' + val)
+        argv = [s.encode("utf-8") for s in argv]
 
         argc = len(argv)
         argc = c.c_int(argc)
@@ -138,7 +143,7 @@ def args_from_config():
     CONFIG['opt_sysimage'] = sysimg = path_option('sysimage', check_exists=True)[0]
     CONFIG['opt_threads'] = int_option('threads', accept_auto=True)[0]
     CONFIG['opt_warn_overwrite'] = choice('warn_overwrite', ['yes', 'no'])[0]
-    CONFIG['opt_handle_signals'] = choice('handle_signals', ['yes', 'no'], default='no')[0]
+    CONFIG['opt_handle_signals'] = choice('handle_signals', ['yes', 'no'])[0]
     CONFIG['opt_startup_file'] = choice('startup_file', ['yes', 'no'])[0]
 
     # Stop if we already initialised
@@ -225,6 +230,24 @@ def jlstr(x):
 
     CONFIG['inited'] = True
 
+    if CONFIG['opt_handle_signals'] is None:
+        if Base.Threads.nthreads() > 1:
+            # a warning to help multithreaded users
+            # TODO: should we set PYTHON_JULIACALL_HANDLE_SIGNALS=yes whenever PYTHON_JULIACALL_THREADS != 1?
+            warnings.warn(
+                "Julia was started with multiple threads "
+                "but multithreading support is experimental in JuliaCall. "
+                "It is recommended to restart Python with the environment variable "
+                "PYTHON_JULIACALL_HANDLE_SIGNALS=yes "
+                "set, otherwise you may experience segfaults or other crashes. "
+                "Note however that this interferes with Python's own signal handling, "
+                "so for example Ctrl-C will not raise KeyboardInterrupt. "
+                "See https://juliapy.github.io/PythonCall.jl/stable/faq/#Is-PythonCall/JuliaCall-thread-safe? "
+                "for further information. "
+                "You can suppress this warning by setting "
+                "PYTHON_JULIACALL_HANDLE_SIGNALS=no."
+            )
+
 init()
 
 def load_ipython_extension(ip):

docs/src/releasenotes.md

@@ -51,7 +51,7 @@ def julia(self, line, cell=None):
         # run the code
         ans = Main.seval('begin\n' + code + '\nend')
         # flush stderr/stdout
-        PythonCall._flush_stdio()
+        PythonCall._ipython._flush_stdio()
         # copy variables back to Python
         # only copy those which are new or have changed value
         for k in outvars + syncvars:
@@ -74,18 +74,22 @@ def load_ipython_extension(ip):
             end
         end""")
     else:
+        # In Julia 1.7+ redirect_stdout() returns a Pipe object. Earlier versions of Julia
+        # just return a tuple of the two pipe ends. This is why we have [1] and [2] below.
+        # They can be dropped on earlier versions.
         PythonCall.seval("""module _ipython
+            using ..PythonCall
             const _redirected_stdout = redirect_stdout()
             const _redirected_stderr = redirect_stderr()
             const _py_stdout = PyIO(pyimport("sys" => "stdout"); line_buffering=true)
             const _py_stderr = PyIO(pyimport("sys" => "stderr"); line_buffering=true)
-            const _redirect_stdout_task = @async write($_py_stdout, $_redirected_stdout)
-            const _redirect_stderr_task = @async write($_py_stderr, $_redirected_stderr)
+            const _redirect_stdout_task = @async write($_py_stdout, $_redirected_stdout[1])
+            const _redirect_stderr_task = @async write($_py_stderr, $_redirected_stderr[1])
             function _flush_stdio()
                 flush(stderr)
                 flush(stdout)
-                flush(_redirected_stderr)
-                flush(_redirected_stdout)
+                flush(_redirected_stderr[2])
+                flush(_redirected_stdout[2])
                 flush(_py_stderr)
                 flush(_py_stdout)
                 nothing

pysrc/juliacall/__init__.py

@@ -17,3 +17,22 @@ def test_issue_394():
     assert jl.f is f
     assert jl.y is y
     assert jl.seval("f(x)") == 4
+
+def test_issue_433():
+    "https://github.com/JuliaPy/PythonCall.jl/issues/433"
+    from juliacall import Main as jl
+
+    # Smoke test
+    jl.seval("x=1\nx=1")
+    assert jl.x == 1
+
+    # Do multiple things
+    out = jl.seval(
+        """
+        function _issue_433_g(x)
+            return x^2
+        end
+        _issue_433_g(5)
+        """
+    )
+    assert out == 25

pysrc/juliacall/ipython.py

@@ -8,7 +8,7 @@ module Compat
     using ..Core
     using ..Core: C, Utils, pynew, incref, getptr, pycopy!, pymodulehooks, pyisnull, pybytes_asvector, pysysmodule, pyosmodule, pystr_fromUTF8
     using ..Convert: pyconvert, @pyconvert
-    using ..Wrap: PyPandasDataFrame
+    using ..Wrap: PyArray, PyPandasDataFrame
     using Serialization: Serialization, AbstractSerializer, serialize, deserialize
     using Tables: Tables
     using Requires: @require

pytest/test_all.py

@@ -42,3 +42,19 @@ function Serialization.serialize(s::AbstractSerializer, x::PyException)
 end
 
 Serialization.deserialize(s::AbstractSerializer, ::Type{PyException}) = PyException(deserialize_py(s))
+
+### PyArray
+#
+# This type holds a pointer and a handle (usually a python memoryview or capsule) which are
+# not serializable by default, and even if they were would not be consistent after
+# serializing each field independently. So we just serialize the wrapped Python object.
+
+function Serialization.serialize(s::AbstractSerializer, x::PyArray)
+    Serialization.serialize_type(s, typeof(x), false)
+    serialize_py(s, x.py)
+end
+
+function Serialization.deserialize(s::AbstractSerializer, ::Type{T}) where {T<:PyArray}
+    # TODO: set buffer and array args too?
+    T(deserialize_py(s); copy=false)
+end

src/Compat/Compat.jl

@@ -23,7 +23,8 @@ Add a new conversion rule for `pyconvert`.
 ### Arguments
 
 - `tname` is a string of the form `"__module__:__qualname__"` identifying a Python type `t`,
-  such as `"builtins:dict"`. This rule only applies to Python objects of this type.
+  such as `"builtins:dict"` or `"sympy.core.symbol:Symbol"`. This rule only applies to
+  Python objects of this type.
 - `T` is a Julia type, such that this rule only applies when the target type intersects
   with `T`.
 - `func` is the function implementing the rule.

src/Compat/serialization.jl

@@ -11,7 +11,7 @@ function init_stdlib()
         # This uses some internals, but Base._start() gets the state more like Julia
         # is if you call the executable directly, in particular it creates workers when
         # the --procs argument is given.
-        push!(Core.ARGS, joinpath(ROOT_DIR, "pysrc", "juliacall", "init.jl"))
+        push!(Base.Core.ARGS, joinpath(ROOT_DIR, "pysrc", "juliacall", "init.jl"))
         Base._start()
         Base.eval(:(PROGRAM_FILE = ""))
 

src/Convert/pyconvert.jl

@@ -10,7 +10,7 @@ function pyjlmodule_dir(self::Module)
 end
 
 function pyjlmodule_seval(self::Module, expr::Py)
-    Py(Base.eval(self, Meta.parse(strip(pyconvert(String, expr)))))
+    Py(Base.eval(self, Meta.parseall(strip(pyconvert(String, expr)))))
 end
 
 function init_module()

src/Core/stdlib.jl

@@ -69,6 +69,19 @@
         @test y == [1, 2, 3]
         @test z == [1, 2, 3]
     end
+    @testset "serialize" begin
+        using Serialization: serialize, deserialize
+        io = IOBuffer()
+        serialize(io, y)
+        seekstart(io)
+        y2 = deserialize(io)
+        @test typeof(y) == typeof(y2)
+        @test eltype(y) == eltype(y2)
+        @test ndims(y) == ndims(y2)
+        @test size(y) == size(y2)
+        @test strides(y) == strides(y2)
+        @test y == y2
+    end
 end
 
 @testitem "PyDict" begin