mix lumen.otp.log.parse

`mix lumen.otp.log.parse` can parse the output of `lumen::otp` tests and classify the errors into a CSV. (Actually tab-separated as some error messages contain commas.)

Author: KronicDeth

native_implemented/otp/tests/log_parser/.formatter.exs

@@ -0,0 +1,4 @@
+# Used by "mix format"
+[
+  inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"]
+]

native_implemented/otp/tests/log_parser/.gitignore

@@ -0,0 +1,29 @@
+# The directory Mix will write compiled artifacts to.
+/_build/
+
+# If you run "mix test --cover", coverage assets end up here.
+/cover/
+
+# The directory Mix downloads your dependencies sources to.
+/deps/
+
+# Where third-party dependencies like ExDoc output generated docs.
+/doc/
+
+# Ignore .fetch files in case you like to edit your project deps locally.
+/.fetch
+
+# If the VM crashes, it generates a dump, let's ignore it too.
+erl_crash.dump
+
+# Also ignore archive artifacts (built via "mix archive.build").
+*.ez
+
+# Ignore package tarball (built via "mix hex.build").
+lumen_otp_log_parser-*.tar
+
+
+# Temporary files for e.g. tests
+/tmp
+
+/test.log.csv

native_implemented/otp/tests/log_parser/README.md

@@ -0,0 +1,8 @@
+# Lumen.OTP.Log.Parser
+
+Parses the log from `cargo test`
+
+1. `cd ../../../..`
+2. Run tests and put ANSI-stripped output to `test.log`: `cargo make test -- --package liblumen_otp lumen::otp:: 2&>1 | sed 's/\x1b\[[0-9;]*m//g' | tee test.log`  
+1. `cd native_implemented/otp/tests/log_parser`
+3. Parse those log into CSV: `mix lumen.otp.log.parse ../../../../test.log > test.log.csv`

native_implemented/otp/tests/log_parser/lib/lumen/otp/log/parser.ex

@@ -0,0 +1,264 @@
+defmodule Lumen.OTP.Log.Parser do
+  @moduledoc """
+  Documentation for `Lumen.OTP.Log.Parser`.
+  """
+
+  def parse(path) do
+    path
+    |> File.stream!()
+      # drop until the first `failures:\n` that is the start of the failure details
+    |> Stream.drop_while(fn
+      "failures:\n" -> false
+      _ -> true
+    end)
+    |> Stream.drop(2)
+      # take while the second `failures:\n` that is summary of test names is shown and details end
+    |> Stream.take_while(fn
+      "failures:\n" -> false
+      _ -> true
+    end)
+    |> Stream.filter(fn
+      # remove common status code
+      "Status code: 101\n" -> false
+      # remove common signal (none)
+      "Signal: ," <> _ -> false
+      _ -> true
+    end)
+    |> Stream.chunk_while(
+         {nil, []},
+         fn
+           "---- lumen::otp::" <> suffix, {previous_test, previous_test_lines} ->
+             test =
+               suffix
+               |> String.split(" ", parts: 2)
+               |> hd()
+
+             {:cont, {previous_test, Enum.reverse(previous_test_lines)}, {test, []}}
+
+           line, {test, acc_lines} ->
+             {:cont, {test, [line | acc_lines]}}
+         end,
+         fn {previous_test, previous_test_lines} ->
+           {:cont, {previous_test, Enum.reverse(previous_test_lines)}, {nil, []}}
+         end
+       )
+    |> Stream.drop(1)
+    |> Stream.map(fn {test, lines} ->
+      case classify(lines) do
+        {:ok, classification} ->
+          {test, classification}
+
+        :error ->
+          Mix.shell().error("Could not classify #{test} error:\n#{Enum.join(lines)}")
+          System.halt(1)
+      end
+    end)
+    |> Enum.sort()
+    |> Enum.each(fn {test, classification} ->
+      Mix.shell().info("#{test}\t#{classification}")
+    end)
+  end
+
+  defp classify([]) do
+    :error
+  end
+
+  defp classify([" right: `SourceId" <> _ = line | tail]) do
+    if String.contains?(line, "source spans cannot start and end in different files") do
+      {:ok, "source spans cannot start and end in different files"}
+    else
+      classify(tail)
+    end
+  end
+
+  defp classify([~S|Assertion failed: (*this && "isa<> used on a null type.")| <> _ | _]) do
+    {:ok, "isa<> used on a null type"}
+  end
+
+  defp classify([
+    ~S|Assertion failed: (funcOp.isExternal() && "cannot define a function more than once"| <>
+    _
+    | _
+  ]) do
+    {:ok, "cannot define a function more than once"}
+  end
+
+  defp classify(["Assertion failed: (hasVal), function getValue, " <> suffix | tail]) do
+    if String.contains?(suffix, "llvm/ADT/Optional.h") do
+      {:ok, "optional does not have value"}
+    else
+      classify(tail)
+    end
+  end
+
+  defp classify(["error: attribute is already defined\n" | _]) do
+    {:ok, "attribute is already defined"}
+  end
+
+  defp classify(["error: could not find file\n" | _]) do
+    {:ok, "could not find file"}
+  end
+
+  defp classify(["error: could not resolve variable\n" | _]) do
+    {:ok, "could not resolve variable"}
+  end
+
+  defp classify(["error: 'eir.binary.match.integer' op attribute 'unit' failed to satisfy constraint: 64-bit signless integer attribute\n" | _]) do
+    {:ok, "'eir.binary.match.integer' op attribute 'unit' failed to satisfy constraint: 64-bit signless integer attribute"}
+  end
+
+  defp classify([
+    "error: 'eir.logical.and' op result #0 must be 1-bit signless integer, but got '!eir.bool'\n"
+    | _
+  ]) do
+    {:ok, "'eir.logical.and' op result must be 1-bit signless integer, but got '!eir.bool'"}
+  end
+
+  defp classify([
+    "error: 'eir.logical.or' op result #0 must be 1-bit signless integer, but got '!eir.bool'\n"
+    | _
+  ]) do
+    {:ok, "'eir.logical.or' op result must be 1-bit signless integer, but got '!eir.bool'"}
+  end
+
+  defp classify([
+    "error: 'eir.map.contains' op operand #1 must be dynamic term type, but got '!eir.atom'\n"
+    | _
+  ]) do
+    {:ok, "'eir.map.contains' op operand #1 must be dynamic term type"}
+  end
+
+  defp classify([
+    <<"error: 'eir.map.insert' op operand #", _, " must be dynamic term type">> <> _ | _
+  ]) do
+    {:ok, "'eir.map.insert' op operand #0 must be dynamic term type"}
+  end
+
+  defp classify(["error: 'eir.map.update' op operand #0 must be dynamic term type, but got '!eir.box<!eir.map>'\n" | _]) do
+    {:ok, "'eir.map.update' op operand #0 must be dynamic term type"}
+  end
+
+  defp classify([
+    "error: 'eir.throw' op operand #2 must be opaque trace reference, but got '!eir.term'\n"
+    | _
+  ]) do
+    {:ok, "'eir.throw' op operand #2 must be opaque trace reference, but got '!eir.term'"}
+  end
+
+  defp classify(["error: invalid cast type, source type is unsupported\n" | _]) do
+    {:ok, "invalid cast type, source type is unsupported"}
+  end
+
+  defp classify(["error: invalid const expression\n" | _]) do
+    {:ok, "invalid const expression"}
+  end
+
+  defp classify(["error: invalid string escape\n" | _]) do
+    {:ok, "invalid string escape"}
+  end
+
+  defp classify(["error: invalid tuple type element" <> _ | _]) do
+    {:ok, "invalid tuple type element"}
+  end
+
+  defp classify([<<"error: operand #", _, " does not dominate this use">> <> _ | _]) do
+    {:ok, "operand does not dominate this use"}
+  end
+
+  defp classify(["error: operand type '!eir.nil' and result type '!eir.box<!eir.cons>' are not cast compatible\n" | _]) do
+    {:ok, "operand type '!eir.nil' and result type '!eir.box<!eir.cons>' are not cast compatible"}
+  end
+
+  defp classify(["error: redefinition of symbol" <> _ | _]) do
+    {:ok, "redefinition of symbol"}
+  end
+
+  defp classify(["error: undefined macro\n" | _]) do
+    {:ok, "undefined macro"}
+  end
+
+  defp classify(["error: unrecognized token\n" | _]) do
+    {:ok, "unrecognized token"}
+  end
+
+  defp classify(["stderr: error: invalid input file" <> _ | _]) do
+    {:ok, "invalid input file"}
+  end
+
+  defp classify(["thread '<unknown>' has overflowed its stack\n" | _]) do
+    {:ok, "stack overflow"}
+  end
+
+  defp classify(["thread '<unnamed>' panicked at 'expected constant const_value" <> suffix | tail]) do
+    [_, after_number] = String.split(suffix, " ", parts: 2)
+
+    case after_number do
+      "to be an atom'" <> _ -> {:ok, "expected constant to be an atom"}
+      _ -> classify(tail)
+    end
+  end
+
+  defp classify(["thread '<unnamed>' panicked at 'expected primop value'" <> _ | _]) do
+    {:ok, "expected primop value"}
+  end
+
+  defp classify(["thread '<unnamed>' panicked at 'expected value, but got pseudo-value'" <> _ | _]) do
+    {:ok, "expected value, but go pseudo-value"}
+  end
+
+  defp classify([
+    "thread '<unnamed>' panicked at 'invalid operation kind: UnpackValueList" <> _ | _
+  ]) do
+    {:ok, "invalid operation kind: UnpackValueList"}
+  end
+
+  defp classify(["thread '<unnamed>' panicked at 'no entry found for key', " <> suffix | tail]) do
+    if String.ends_with?(suffix, "libeir_syntax_erl/src/lower/expr/record.rs:138:19\n") do
+      {:ok, "no entry found for key in record"}
+    else
+      classify(tail)
+    end
+  end
+
+  defp classify(["thread '<unnamed>' panicked at 'not implemented', " <> suffix | tail]) do
+    if String.ends_with?(suffix, "libeir_syntax_erl/src/lower/expr/comprehension.rs:130:44\n") do
+      {:ok, "binary generators not implemented"}
+    else
+      classify(tail)
+    end
+  end
+
+  defp classify([
+    "thread '<unnamed>' panicked at 'not yet implemented: unimplemented call type LocalDynamic" <>
+    _
+    | _
+  ]) do
+    {:ok, "unimplemented call type LocalDynamic"}
+  end
+
+  defp classify(["thread '<unnamed>' panicked at 'the given value is not a known block" <> _ | _]) do
+    {:ok, "the given value is not a known block"}
+  end
+
+  defp classify(["thread 'lumen::otp::" <> suffix | tail]) do
+    if String.contains?(suffix, "Compilation timed out") do
+      {:ok, "compilation timed out"}
+    else
+      classify(tail)
+    end
+  end
+
+  defp classify(["Undefined symbols" <> _ | _]) do
+    {:ok, "undefined symbols"}
+  end
+
+  defp classify(["warning: invalid compile option\n" | _]) do
+    {:ok, "invalid compile option"}
+  end
+
+  defp classify(["stdout: TODO file directive -file" <> _ | _]) do
+    {:ok, "file directive (-file) unimplemented"}
+  end
+
+  defp classify([_ | tail]), do: classify(tail)
+end

native_implemented/otp/tests/log_parser/lib/mix/tasks/lumen/otp/log/parse.ex

@@ -0,0 +1,18 @@
+defmodule Mix.Tasks.Lumen.Otp.Log.Parse do
+  @moduledoc "Parses the test.log outputted from `cargo test --package liblumen_otp -- lumen::otp`"
+  @shortdoc "Parses test.log"
+
+  alias Lumen.OTP.Log.Parser
+
+  use Mix.Task
+
+  @impl Mix.Task
+
+  def run([path]) do
+    Parser.parse(path)
+  end
+
+  def run(_) do
+    Mix.shell().info("mix lumen.otp.log.parse LOG_PATH")
+  end
+end

native_implemented/otp/tests/log_parser/mix.exs

@@ -0,0 +1,28 @@
+defmodule Lumen.OTP.Log.Parser.MixProject do
+  use Mix.Project
+
+  def project do
+    [
+      app: :lumen_otp_log_parser,
+      version: "0.1.0",
+      elixir: "~> 1.11",
+      start_permanent: Mix.env() == :prod,
+      deps: deps()
+    ]
+  end
+
+  # Run "mix help compile.app" to learn about applications.
+  def application do
+    [
+      extra_applications: [:logger]
+    ]
+  end
+
+  # Run "mix help deps" to learn about dependencies.
+  defp deps do
+    [
+      # {:dep_from_hexpm, "~> 0.3.0"},
+      # {:dep_from_git, git: "https://github.com/elixir-lang/my_dep.git", tag: "0.1.0"}
+    ]
+  end
+end

native_implemented/otp/tests/log_parser/test/lumen/otp/log/parser_test.exs

@@ -0,0 +1,8 @@
+defmodule Lumen.OTP.Log.ParserTest do
+  use ExUnit.Case
+  doctest Lumen.OTP.Log.Parser
+
+  test "greets the world" do
+    assert Lumen.OTP.Log.Parser.hello() == :world
+  end
+end

native_implemented/otp/tests/log_parser/test/test_helper.exs

@@ -0,0 +1 @@
+ExUnit.start()