From 74afeece8cd8585d1105ce1cb22c4e38677270bd Mon Sep 17 00:00:00 2001 From: sabiwara Date: Wed, 27 May 2026 08:52:12 +0900 Subject: [PATCH] Never suggest back invalid characters in tokenizer hint --- lib/elixir/src/elixir_tokenizer.erl | 2 +- lib/elixir/test/elixir/kernel/parser_test.exs | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/elixir/src/elixir_tokenizer.erl b/lib/elixir/src/elixir_tokenizer.erl index 0b92542a9f..6592aed98c 100644 --- a/lib/elixir/src/elixir_tokenizer.erl +++ b/lib/elixir/src/elixir_tokenizer.erl @@ -1438,7 +1438,7 @@ suggest_simpler_unexpected_token_in_error(Wrong, Line, WrongColumn, Scope) -> {error, _Reason} -> ConfusableSkeleton = 'Elixir.String.Tokenizer.Security':confusable_skeleton(Wrong), case (Scope#elixir_tokenizer.identifier_tokenizer):tokenize(ConfusableSkeleton) of - {_, Simpler, _, _, _, _} -> + {_, Simpler, _, _, _, _} when Simpler =/= Wrong -> Message = suggest_change("Codepoint failed identifier tokenization, but a simpler form was found.", Wrong, "You could write the above in a similar way that is accepted by Elixir:", diff --git a/lib/elixir/test/elixir/kernel/parser_test.exs b/lib/elixir/test/elixir/kernel/parser_test.exs index 9c7e56d382..bf37abb5d2 100644 --- a/lib/elixir/test/elixir/kernel/parser_test.exs +++ b/lib/elixir/test/elixir/kernel/parser_test.exs @@ -1353,6 +1353,12 @@ defmodule Kernel.ParserTest do ] assert_syntax_error(message, ~c"fooی𝚳") + + # regression test: ǜ (should not suggest back the wrong character) + assert_syntax_error( + ["nofile:1:4:", ~s/unexpected token: "#{"\u01DC"}" (column 4, code point U+01DC)/], + ~c":fooǜ" + ) end test "keyword missing space" do