Skip to content

Make sure we never suggest back invalid characters #15421

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
josevalim merged 1 commit into from
May 27, 2026
+7 −1
Merged

Make sure we never suggest back invalid characters #15421

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion lib/elixir/src/elixir_tokenizer.erl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1438,7 +1438,7 @@ suggest_simpler_unexpected_token_in_error(Wrong, Line, WrongColumn, Scope) ->
{error, _Reason} ->
ConfusableSkeleton = 'Elixir.String.Tokenizer.Security':confusable_skeleton(Wrong),
case (Scope#elixir_tokenizer.identifier_tokenizer):tokenize(ConfusableSkeleton) of
{_, Simpler, _, _, _, _} ->
{_, Simpler, _, _, _, _} when Simpler =/= Wrong ->
Message = suggest_change("Codepoint failed identifier tokenization, but a simpler form was found.",
Wrong,
"You could write the above in a similar way that is accepted by Elixir:",
Expand Down
6 changes: 6 additions & 0 deletions lib/elixir/test/elixir/kernel/parser_test.exs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1353,6 +1353,12 @@ defmodule Kernel.ParserTest do
]

assert_syntax_error(message, ~c"fooی𝚳")

# regression test: ǜ (should not suggest back the wrong character)
assert_syntax_error(
["nofile:1:4:", ~s/unexpected token: "#{"\u01DC"}" (column 4, code point U+01DC)/],
~c":fooǜ"
)
end

test "keyword missing space" do
Expand Down
Loading