Allow sigil modifiers to contain digits (#11283)

Author: dannote

lib/elixir/src/elixir_tokenizer.erl

@@ -951,7 +951,7 @@ unsafe_to_atom(List, Line, Column, #elixir_tokenizer{existing_atoms_only=true})
 unsafe_to_atom(List, _Line, _Column, #elixir_tokenizer{}) when is_list(List) ->
   {ok, list_to_atom(List)}.
 
-collect_modifiers([H | T], Buffer) when ?is_downcase(H) or ?is_upcase(H) ->
+collect_modifiers([H | T], Buffer) when ?is_downcase(H) or ?is_upcase(H) or ?is_digit(H) ->
   collect_modifiers(T, [H | Buffer]);
 
 collect_modifiers(Rest, Buffer) ->

lib/elixir/test/erlang/tokenizer_test.erl

@@ -264,7 +264,8 @@ sigil_terminator_test() ->
    {identifier, {1, 12, nil}, bar}] = tokenize("~r/foo/ == bar"),
   [{sigil, {1, 1, nil}, 114, [<<"foo">>], "iu", nil, <<"/">>},
    {comp_op, {1, 11, nil}, '=='},
-   {identifier, {1, 14, nil}, bar}] = tokenize("~r/foo/iu == bar").
+   {identifier, {1, 14, nil}, bar}] = tokenize("~r/foo/iu == bar"),
+  [{sigil, {1, 1, nil}, 77, [<<"1 2 3">>], "u8", nil, <<"[">>}] = tokenize("~M[1 2 3]u8").
 
 sigil_heredoc_test() ->
   [{sigil, {1, 1, nil}, 83, [<<"sigil heredoc\n">>], "", 0, <<"\"\"\"">>}] = tokenize("~S\"\"\"\nsigil heredoc\n\"\"\""),