Load modules lazily and only if needed

lib/elixir/lib/kernel/error_handler.ex

@@ -41,7 +41,15 @@ defmodule Kernel.ErrorHandler do
     :erlang.garbage_collect(self())
 
     receive do
-      {^ref, value} -> value
+      {^ref, {:loading, pid}} ->
+        ref = :erlang.monitor(:process, pid)
+
+        receive do
+          {:DOWN, ^ref, _, _, _} -> :found
+        end
+
+      {^ref, value} ->
+        value
     end
   end
 end

lib/elixir/lib/kernel/parallel_compiler.ex

@@ -198,7 +198,7 @@ defmodule Kernel.ParallelCompiler do
           {:ok, [atom], [warning] | info()}
           | {:error, [error] | [Code.diagnostic(:error)], [warning] | info()}
   def compile_to_path(files, path, options \\ []) when is_binary(path) and is_list(options) do
-    spawn_workers(files, {:compile, path}, options)
+    spawn_workers(files, {:compile, path}, Keyword.put(options, :dest, path))
   end
 
   @doc """
@@ -338,8 +338,11 @@ defmodule Kernel.ParallelCompiler do
   end
 
   defp write_module_binaries(result, {:compile, path}, timestamp) do
+    File.mkdir_p!(path)
+    Code.prepend_path(path)
+
     Enum.flat_map(result, fn
-      {{:module, module}, binary} when is_binary(binary) ->
+      {{:module, module}, {binary, _}} when is_binary(binary) ->
         full_path = Path.join(path, Atom.to_string(module) <> ".beam")
         File.write!(full_path, binary)
         if timestamp, do: File.touch!(full_path, timestamp)
@@ -351,7 +354,7 @@ defmodule Kernel.ParallelCompiler do
   end
 
   defp write_module_binaries(result, _output, _timestamp) do
-    for {{:module, module}, binary} when is_binary(binary) <- result, do: module
+    for {{:module, module}, {binary, _}} when is_binary(binary) <- result, do: module
   end
 
   ## Verification
@@ -366,7 +369,7 @@ defmodule Kernel.ParallelCompiler do
 
   defp maybe_check_modules(result, runtime_modules, state) do
     compiled_modules =
-      for {{:module, module}, binary} when is_binary(binary) <- result,
+      for {{:module, module}, {binary, _}} when is_binary(binary) <- result,
           do: module
 
     profile(
@@ -438,10 +441,10 @@ defmodule Kernel.ParallelCompiler do
         :erlang.put(:elixir_compiler_file, file)
 
         try do
-          case output do
-            {:compile, path} -> compile_file(file, path, parent)
-            :compile -> compile_file(file, dest, parent)
-            :require -> require_file(file, parent)
+          if output == :require do
+            require_file(file, parent)
+          else
+            compile_file(file, dest, parent)
           end
         catch
           kind, reason ->
@@ -582,7 +585,7 @@ defmodule Kernel.ParallelCompiler do
   end
 
   defp count_modules(result) do
-    Enum.count(result, &match?({{:module, _}, binary} when is_binary(binary), &1))
+    Enum.count(result, &match?({{:module, _}, {binary, _}} when is_binary(binary), &1))
   end
 
   defp each_cycle_return({kind, modules, warnings}), do: {kind, modules, warnings}
@@ -649,19 +652,51 @@ defmodule Kernel.ParallelCompiler do
           state
         )
 
-      {:module_available, child, ref, file, module, binary} ->
+      {:load_module?, child, ref, module} ->
+        # If compiling files to disk, we only load the module
+        # if other modules are waiting for it.
+        load? =
+          case state.output do
+            {:compile, _} -> match?(%{{:module, ^module} => [_ | _]}, result)
+            _ -> true
+          end
+
+        send(child, {ref, load?})
+        spawn_workers(queue, spawned, waiting, files, result, warnings, errors, state)
+
+      {{:module_loaded, module}, _ref, _type, _pid, _reason} ->
+        result =
+          Map.update!(result, {:module, module}, fn {binary, _loader} -> {binary, true} end)
+
+        spawn_workers(queue, spawned, waiting, files, result, warnings, errors, state)
+
+      {:module_available, child, ref, file, module, binary, loaded?} ->
         state.each_module.(file, module, binary)
 
+        {available, load_status} =
+          case Map.get(result, {:module, module}) do
+            # We prefer to load in the client, if possible,
+            # to avoid locking the compilation server.
+            [_ | _] = pids when loaded? ->
+              {Enum.map(pids, &{&1, :found}), loaded?}
+
+            [_ | _] = pids ->
+              pid = load_module(module, binary, state.dest)
+              {Enum.map(pids, &{&1, {:loading, pid}}), pid}
+
+            _ ->
+              {[], loaded?}
+          end
+
         # Release the module loader which is waiting for an ack
         send(child, {ref, :ack})
-        {available, result} = update_result(result, :module, module, binary)
 
         spawn_workers(
           available ++ queue,
           spawned,
           waiting,
           files,
-          result,
+          Map.put(result, {:module, module}, {binary, load_status}),
           warnings,
           errors,
           state
@@ -680,7 +715,9 @@ defmodule Kernel.ParallelCompiler do
 
         {waiting, files, result} =
           if not is_list(available_or_pending) or on in defining do
-            send(child_pid, {ref, :found})
+            # If what we are waiting on was defined but not loaded, we do it now.
+            {reply, result} = load_pending(kind, on, result, state)
+            send(child_pid, {ref, reply})
             {waiting, files, result}
           else
             waiting = Map.put(waiting, child_pid, {kind, ref, file_pid, on, defining, deadlock})
@@ -774,6 +811,52 @@ defmodule Kernel.ParallelCompiler do
     {{:error, Enum.reverse(errors, fun.()), info}, state}
   end
 
+  defp load_pending(kind, module, result, state) do
+    case result do
+      %{{:module, ^module} => {binary, load_status}}
+      when kind in [:module, :struct] and is_binary(binary) ->
+        case load_status do
+          true ->
+            {:found, result}
+
+          false ->
+            pid = load_module(module, binary, state.dest)
+            result = Map.put(result, {:module, module}, {binary, pid})
+            {{:loading, pid}, result}
+
+          pid when is_pid(pid) ->
+            {{:loading, pid}, result}
+        end
+
+      _ ->
+        {:found, result}
+    end
+  end
+
+  defp load_module(module, binary, dest) do
+    {pid, _ref} =
+      :erlang.spawn_opt(
+        fn ->
+          beam_location =
+            case dest do
+              nil ->
+                []
+
+              dest ->
+                :filename.join(
+                  :elixir_utils.characters_to_list(dest),
+                  Atom.to_charlist(module) ++ ~c".beam"
+                )
+            end
+
+          :code.load_binary(module, beam_location, binary)
+        end,
+        monitor: [tag: {:module_loaded, module}]
+      )
+
+    pid
+  end
+
   defp update_result(result, kind, module, value) do
     available =
       case Map.get(result, {kind, module}) do

lib/elixir/lib/module.ex

@@ -421,6 +421,12 @@ defmodule Module do
         end
       end
 
+  The function given to `on_load` should avoid calling functions from
+  other modules. If you must call functions in other modules and those
+  modules are defined within the same project, the called modules must
+  have the `@compile {:autoload, true}` annotation, so they are loaded
+  upfront (and not from within the `@on_load` callback).
+
   ### `@vsn`
 
   Specify the module version. Accepts any valid Elixir value, for example:
@@ -590,9 +596,11 @@ defmodule Module do
       name/arity pairs. Inlining is applied locally, calls from another
       module are not affected by this option
 
-    * `@compile {:autoload, false}` - disables automatic loading of
-      modules after compilation. Instead, the module will be loaded after
-      it is dispatched to
+    * `@compile {:autoload, true}` - configures if modules are automatically
+      loaded after definition. It defaults to `false` when compiling modules
+      to `.beam` files in disk (as the modules are then lazily loaded from
+      disk). If modules are not compiled to disk, then they are always loaded,
+      regardless of this flag
 
     * `@compile {:no_warn_undefined, Mod}` or
       `@compile {:no_warn_undefined, {Mod, fun, arity}}` - does not warn if

lib/elixir/src/elixir_erl_compiler.erl

@@ -8,6 +8,7 @@
 
 spawn(Fun) ->
   CompilerInfo = get(elixir_compiler_info),
+  {error_handler, ErrorHandler} = erlang:process_info(self(), error_handler),
 
   CodeDiagnostics =
     case get(elixir_code_diagnostics) of
@@ -17,6 +18,7 @@ spawn(Fun) ->
 
   {_, Ref} =
     spawn_monitor(fun() ->
+      erlang:process_flag(error_handler, ErrorHandler),
       put(elixir_compiler_info, CompilerInfo),
       put(elixir_code_diagnostics, CodeDiagnostics),
 

lib/elixir/src/elixir_module.erl

@@ -219,17 +219,17 @@ compile(Meta, Module, ModuleAsCharlist, Block, Vars, Prune, E) ->
 
         compile_error_if_tainted(DataSet, E),
         Binary = elixir_erl:compile(ModuleMap),
-        Autoload = proplists:get_value(autoload, CompileOpts, true),
+        Autoload = proplists:get_value(autoload, CompileOpts, false) or load_module(Module),
         spawn_parallel_checker(CheckerInfo, Module, ModuleMap, BeamLocation),
         {Binary, PersistedAttributes, Autoload}
       end),
 
     Autoload andalso code:load_binary(Module, BeamLocation, Binary),
+    make_module_available(Module, Binary, Autoload),
     put_compiler_modules(CompilerModules),
     eval_callbacks(Line, DataBag, after_compile, [CallbackE, Binary], CallbackE),
     elixir_env:trace({on_module, Binary, none}, ModuleE),
     warn_unused_attributes(DataSet, DataBag, PersistedAttributes, E),
-    make_module_available(Module, Binary),
     (element(2, CheckerInfo) == nil) andalso
       [VerifyMod:VerifyFun(Module) ||
        {VerifyMod, VerifyFun} <- bag_lookup_element(DataBag, {accumulate, after_verify}, 2)],
@@ -574,7 +574,7 @@ spawn_parallel_checker(CheckerInfo, Module, ModuleMap, BeamLocation) ->
     end,
   'Elixir.Module.ParallelChecker':spawn(CheckerInfo, Module, ModuleMap, BeamLocation, Log).
 
-make_module_available(Module, Binary) ->
+make_module_available(Module, Binary, Loaded) ->
   case get(elixir_module_binaries) of
     Current when is_list(Current) ->
       put(elixir_module_binaries, [{Module, Binary} | Current]);
@@ -587,10 +587,20 @@ make_module_available(Module, Binary) ->
       ok;
     {PID, _} ->
       Ref = make_ref(),
-      PID ! {module_available, self(), Ref, get(elixir_compiler_file), Module, Binary},
+      PID ! {module_available, self(), Ref, get(elixir_compiler_file), Module, Binary, Loaded},
       receive {Ref, ack} -> ok end
   end.
 
+load_module(Module) ->
+  case get(elixir_compiler_info) of
+    undefined ->
+      true;
+    {PID, _} ->
+      Ref = make_ref(),
+      PID ! {'load_module?', self(), Ref, Module},
+      receive {Ref, Boolean} -> Boolean end
+  end.
+
 %% Error handling and helpers.
 
 %% We've reached the elixir_module or eval internals, skip it with the rest

lib/mix/lib/mix/compilers/elixir.ex

@@ -180,9 +180,6 @@ defmodule Mix.Compilers.Elixir do
       end
 
       Mix.Project.ensure_structure()
-
-      # We don't want to cache this path as we will write to it
-      true = Code.prepend_path(dest)
       previous_opts = set_compiler_opts(opts)
 
       try do