Load modules lazily

Author: josevalim

lib/elixir/lib/kernel/parallel_compiler.ex

@@ -181,7 +181,7 @@ defmodule Kernel.ParallelCompiler do
           {:ok, [atom], [warning] | info()}
           | {:error, [error] | [Code.diagnostic(:error)], [warning] | info()}
   def compile_to_path(files, path, options \\ []) when is_binary(path) and is_list(options) do
-    spawn_workers(files, {:compile, path}, options)
+    spawn_workers(files, {:compile, path}, Keyword.put(options, :dest, path))
   end
 
   @doc """
@@ -320,6 +320,9 @@ defmodule Kernel.ParallelCompiler do
   end
 
   defp write_module_binaries(result, {:compile, path}, timestamp) do
+    File.mkdir_p!(path)
+    Code.prepend_path(path)
+
     Enum.flat_map(result, fn
       {{:module, module}, binary} when is_binary(binary) ->
         full_path = Path.join(path, Atom.to_string(module) <> ".beam")
@@ -420,8 +423,8 @@ defmodule Kernel.ParallelCompiler do
 
         try do
           case output do
-            {:compile, path} -> compile_file(file, path, parent)
-            :compile -> compile_file(file, dest, parent)
+            {:compile, _} -> compile_file(file, dest, false, parent)
+            :compile -> compile_file(file, dest, true, parent)
             :require -> require_file(file, parent)
           end
         catch
@@ -527,9 +530,9 @@ defmodule Kernel.ParallelCompiler do
     wait_for_messages([], spawned, waiting, files, result, warnings, errors, state)
   end
 
-  defp compile_file(file, path, parent) do
+  defp compile_file(file, path, force_load?, parent) do
     :erlang.process_flag(:error_handler, Kernel.ErrorHandler)
-    :erlang.put(:elixir_compiler_dest, path)
+    :erlang.put(:elixir_compiler_dest, {path, force_load?})
     :elixir_compiler.file(file, &each_file(&1, &2, parent))
   end
 
@@ -630,19 +633,28 @@ defmodule Kernel.ParallelCompiler do
           state
         )
 
-      {:module_available, child, ref, file, module, binary} ->
+      {:module_available, child, ref, file, module, binary, loaded?} ->
         state.each_module.(file, module, binary)
 
+        available =
+          case Map.get(result, {:module, module}) do
+            [_ | _] = pids ->
+              loaded? or load_module(module, binary, state)
+              Enum.map(pids, &{&1, :found})
+
+            _ ->
+              []
+          end
+
         # Release the module loader which is waiting for an ack
         send(child, {ref, :ack})
-        {available, result} = update_result(result, :module, module, binary)
 
         spawn_workers(
           available ++ queue,
           spawned,
           waiting,
           files,
-          result,
+          Map.put(result, {:module, module}, binary),
           warnings,
           errors,
           state
@@ -660,14 +672,31 @@ defmodule Kernel.ParallelCompiler do
         available_or_pending = Map.get(result, {kind, on}, [])
 
         {waiting, files, result} =
-          if not is_list(available_or_pending) or on in defining do
-            send(child_pid, {ref, :found})
-            {waiting, files, result}
-          else
-            waiting = Map.put(waiting, child_pid, {kind, ref, file_pid, on, defining, deadlock})
-            files = update_timing(files, file_pid, :compiling)
-            result = Map.put(result, {kind, on}, [child_pid | available_or_pending])
-            {waiting, files, result}
+          cond do
+            # TODO: Refactor me
+            kind == :struct and is_binary(Map.get(result, {:module, on})) and
+                not :erlang.module_loaded(on) ->
+              load_module(on, Map.get(result, {:module, on}), state)
+              send(child_pid, {ref, :found})
+              {waiting, files, result}
+
+            # We have the module but it was not loaded yet, so we load it
+            kind == :module and is_binary(available_or_pending) and not :erlang.module_loaded(on) ->
+              load_module(on, available_or_pending, state)
+              send(child_pid, {ref, :found})
+              {waiting, files, result}
+
+            # Whatever we have is available, return it
+            not is_list(available_or_pending) or on in defining ->
+              send(child_pid, {ref, :found})
+              {waiting, files, result}
+
+            # We need to wait for it
+            true ->
+              waiting = Map.put(waiting, child_pid, {kind, ref, file_pid, on, defining, deadlock})
+              files = update_timing(files, file_pid, :compiling)
+              result = Map.put(result, {kind, on}, [child_pid | available_or_pending])
+              {waiting, files, result}
           end
 
         spawn_workers(queue, spawned, waiting, files, result, warnings, errors, state)
@@ -755,6 +784,22 @@ defmodule Kernel.ParallelCompiler do
     {{:error, Enum.reverse(errors, fun.()), info}, state}
   end
 
+  defp load_module(module, binary, state) do
+    beam_location =
+      case state.dest do
+        nil ->
+          []
+
+        dest ->
+          :filename.join(
+            :elixir_utils.characters_to_list(dest),
+            Atom.to_charlist(module) ++ ~c".beam"
+          )
+      end
+
+    :code.load_binary(module, beam_location, binary)
+  end
+
   defp update_result(result, kind, module, value) do
     available =
       case Map.get(result, {kind, module}) do

lib/elixir/src/elixir_erl_compiler.erl

@@ -4,6 +4,7 @@
 
 spawn(Fun) ->
   CompilerInfo = get(elixir_compiler_info),
+  {error_handler, ErrorHandler} = erlang:process_info(self(), error_handler),
 
   CodeDiagnostics =
     case get(elixir_code_diagnostics) of
@@ -13,6 +14,7 @@ spawn(Fun) ->
 
   {_, Ref} =
     spawn_monitor(fun() ->
+      erlang:process_flag(error_handler, ErrorHandler),
       put(elixir_compiler_info, CompilerInfo),
       put(elixir_code_diagnostics, CodeDiagnostics),
 

lib/elixir/src/elixir_module.erl

@@ -155,7 +155,7 @@ compile(Meta, Module, ModuleAsCharlist, Block, Vars, Prune, E) ->
     put_compiler_modules([Module | CompilerModules]),
     {Result, ModuleE, CallbackE} = eval_form(Line, Module, DataBag, Block, Vars, Prune, E),
     CheckerInfo = checker_info(),
-    BeamLocation = beam_location(ModuleAsCharlist),
+    {BeamLocation, Forceload} = beam_location(ModuleAsCharlist),
 
     {Binary, PersistedAttributes, Autoload} =
       elixir_erl_compiler:spawn(fun() ->
@@ -215,17 +215,19 @@ compile(Meta, Module, ModuleAsCharlist, Block, Vars, Prune, E) ->
 
         compile_error_if_tainted(DataSet, E),
         Binary = elixir_erl:compile(ModuleMap),
-        Autoload = proplists:get_value(autoload, CompileOpts, true),
+        %% TODO: Change docs
+        Autoload = proplists:get_value(autoload, CompileOpts, false),
         spawn_parallel_checker(CheckerInfo, Module, ModuleMap),
         {Binary, PersistedAttributes, Autoload}
       end),
 
-    Autoload andalso code:load_binary(Module, BeamLocation, Binary),
+    (Forceload or Autoload) andalso code:load_binary(Module, BeamLocation, Binary),
     put_compiler_modules(CompilerModules),
+    %% TODO: Do not check twice
+    make_module_available(Module, Binary, Forceload or Autoload),
     eval_callbacks(Line, DataBag, after_compile, [CallbackE, Binary], CallbackE),
     elixir_env:trace({on_module, Binary, none}, ModuleE),
     warn_unused_attributes(DataSet, DataBag, PersistedAttributes, E),
-    make_module_available(Module, Binary),
     (element(2, CheckerInfo) == nil) andalso
       [VerifyMod:VerifyFun(Module) ||
        {VerifyMod, VerifyFun} <- bag_lookup_element(DataBag, {accumulate, after_verify}, 2)],
@@ -544,10 +546,12 @@ bag_lookup_element(Table, Name, Pos) ->
 
 beam_location(ModuleAsCharlist) ->
   case get(elixir_compiler_dest) of
-    Dest when is_binary(Dest) ->
-      filename:join(elixir_utils:characters_to_list(Dest), ModuleAsCharlist ++ ".beam");
+    {Dest, ForceLoad} when is_binary(Dest) ->
+      BeamLocation =
+        filename:join(elixir_utils:characters_to_list(Dest), ModuleAsCharlist ++ ".beam"),
+      {BeamLocation, ForceLoad};
     _ ->
-      ""
+      {"", true}
   end.
 
 %% Integration with elixir_compiler that makes the module available
@@ -568,7 +572,7 @@ spawn_parallel_checker(CheckerInfo, Module, ModuleMap) ->
     end,
   'Elixir.Module.ParallelChecker':spawn(CheckerInfo, Module, ModuleMap, Log).
 
-make_module_available(Module, Binary) ->
+make_module_available(Module, Binary, Loaded) ->
   case get(elixir_module_binaries) of
     Current when is_list(Current) ->
       put(elixir_module_binaries, [{Module, Binary} | Current]);
@@ -581,7 +585,7 @@ make_module_available(Module, Binary) ->
       ok;
     {PID, _} ->
       Ref = make_ref(),
-      PID ! {module_available, self(), Ref, get(elixir_compiler_file), Module, Binary},
+      PID ! {module_available, self(), Ref, get(elixir_compiler_file), Module, Binary, Loaded},
       receive {Ref, ack} -> ok end
   end.
 

lib/mix/lib/mix/compilers/elixir.ex

@@ -166,9 +166,6 @@ defmodule Mix.Compilers.Elixir do
 
       Mix.Utils.compiling_n(length(stale), :ex)
       Mix.Project.ensure_structure()
-
-      # We don't want to cache this path as we will write to it
-      true = Code.prepend_path(dest)
       previous_opts = set_compiler_opts(opts)
 
       try do