Auto merge of rust-lang#15303 - oxalica:fix/byte-escape-highlight, r=lowr

bors · bors · commit 99718d0c8bc5 · 2023-07-22T20:25:40.000Z
Fix highlighting of byte escape sequences

Currently non-UTF8 escape sequences in byte strings and any escape sequences in byte literals are ignored.
diff --git a/crates/hir-def/src/body/pretty.rs b/crates/hir-def/src/body/pretty.rs
@@ -634,7 +634,7 @@ impl Printer<'_> {
         match literal {
             Literal::String(it) => w!(self, "{:?}", it),
             Literal::ByteString(it) => w!(self, "\"{}\"", it.escape_ascii()),
-            Literal::CString(it) => w!(self, "\"{}\\0\"", it),
+            Literal::CString(it) => w!(self, "\"{}\\0\"", it.escape_ascii()),
             Literal::Char(it) => w!(self, "'{}'", it.escape_debug()),
             Literal::Bool(it) => w!(self, "{}", it),
             Literal::Int(i, suffix) => {
diff --git a/crates/hir-def/src/hir.rs b/crates/hir-def/src/hir.rs
@@ -85,7 +85,7 @@ impl fmt::Display for FloatTypeWrapper {
 pub enum Literal {
     String(Box<str>),
     ByteString(Box<[u8]>),
-    CString(Box<str>),
+    CString(Box<[u8]>),
     Char(char),
     Bool(bool),
     Int(i128, Option<BuiltinInt>),
diff --git a/crates/hir-ty/src/mir/lower.rs b/crates/hir-ty/src/mir/lower.rs
@@ -1355,7 +1355,6 @@ impl<'ctx> MirLowerCtx<'ctx> {
                 return Ok(Operand::from_concrete_const(data, mm, ty));
             }
             hir_def::hir::Literal::CString(b) => {
-                let b = b.as_bytes();
                 let bytes = b.iter().copied().chain(iter::once(0)).collect::<Vec<_>>();
 
                 let mut data = Vec::with_capacity(mem::size_of::<usize>() * 2);
diff --git a/crates/ide/src/syntax_highlighting.rs b/crates/ide/src/syntax_highlighting.rs
@@ -24,7 +24,7 @@ use syntax::{
 
 use crate::{
     syntax_highlighting::{
-        escape::{highlight_escape_char, highlight_escape_string},
+        escape::{highlight_escape_byte, highlight_escape_char, highlight_escape_string},
         format::highlight_format_string,
         highlights::Highlights,
         macro_::MacroHighlighter,
@@ -471,6 +471,14 @@ fn traverse(
                 };
 
                 highlight_escape_char(hl, &char, range.start())
+            } else if ast::Byte::can_cast(token.kind())
+                && ast::Byte::can_cast(descended_token.kind())
+            {
+                let Some(byte) = ast::Byte::cast(token) else {
+                    continue;
+                };
+
+                highlight_escape_byte(hl, &byte, range.start())
             }
         }
 
diff --git a/crates/ide/src/syntax_highlighting/escape.rs b/crates/ide/src/syntax_highlighting/escape.rs
@@ -1,7 +1,7 @@
 //! Syntax highlighting for escape sequences
 use crate::syntax_highlighting::highlights::Highlights;
 use crate::{HlRange, HlTag};
-use syntax::ast::{Char, IsString};
+use syntax::ast::{Byte, Char, IsString};
 use syntax::{AstToken, TextRange, TextSize};
 
 pub(super) fn highlight_escape_string<T: IsString>(
@@ -10,14 +10,14 @@ pub(super) fn highlight_escape_string<T: IsString>(
     start: TextSize,
 ) {
     string.escaped_char_ranges(&mut |piece_range, char| {
-        if char.is_err() {
-            return;
-        }
-
         if string.text()[piece_range.start().into()..].starts_with('\\') {
+            let highlight = match char {
+                Ok(_) => HlTag::EscapeSequence,
+                Err(_) => HlTag::InvalidEscapeSequence,
+            };
             stack.add(HlRange {
                 range: piece_range + start,
-                highlight: HlTag::EscapeSequence.into(),
+                highlight: highlight.into(),
                 binding_hash: None,
             });
         }
@@ -26,6 +26,9 @@ pub(super) fn highlight_escape_string<T: IsString>(
 
 pub(super) fn highlight_escape_char(stack: &mut Highlights, char: &Char, start: TextSize) {
     if char.value().is_none() {
+        // We do not emit invalid escapes highlighting here. The lexer would likely be in a bad
+        // state and this token contains junks, since `'` is not a reliable delimiter (consider
+        // lifetimes). Nonetheless, parser errors should already be emitted.
         return;
     }
 
@@ -43,3 +46,24 @@ pub(super) fn highlight_escape_char(stack: &mut Highlights, char: &Char, start:
         TextRange::new(start + TextSize::from(1), start + TextSize::from(text.len() as u32 + 1));
     stack.add(HlRange { range, highlight: HlTag::EscapeSequence.into(), binding_hash: None })
 }
+
+pub(super) fn highlight_escape_byte(stack: &mut Highlights, byte: &Byte, start: TextSize) {
+    if byte.value().is_none() {
+        // See `highlight_escape_char` for why no error highlighting here.
+        return;
+    }
+
+    let text = byte.text();
+    if !text.starts_with("b'") || !text.ends_with('\'') {
+        return;
+    }
+
+    let text = &text[2..text.len() - 1];
+    if !text.starts_with('\\') {
+        return;
+    }
+
+    let range =
+        TextRange::new(start + TextSize::from(2), start + TextSize::from(text.len() as u32 + 2));
+    stack.add(HlRange { range, highlight: HlTag::EscapeSequence.into(), binding_hash: None })
+}
diff --git a/crates/ide/src/syntax_highlighting/html.rs b/crates/ide/src/syntax_highlighting/html.rs
@@ -109,6 +109,7 @@ pre                 { color: #DCDCCC; background: #3F3F3F; font-size: 22px; padd
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 ";
diff --git a/crates/ide/src/syntax_highlighting/tags.rs b/crates/ide/src/syntax_highlighting/tags.rs
@@ -29,6 +29,7 @@ pub enum HlTag {
     Comment,
     EscapeSequence,
     FormatSpecifier,
+    InvalidEscapeSequence,
     Keyword,
     NumericLiteral,
     Operator(HlOperator),
@@ -166,6 +167,7 @@ impl HlTag {
             HlTag::CharLiteral => "char_literal",
             HlTag::Comment => "comment",
             HlTag::EscapeSequence => "escape_sequence",
+            HlTag::InvalidEscapeSequence => "invalid_escape_sequence",
             HlTag::FormatSpecifier => "format_specifier",
             HlTag::Keyword => "keyword",
             HlTag::Punctuation(punct) => match punct {
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_assoc_functions.html b/crates/ide/src/syntax_highlighting/test_data/highlight_assoc_functions.html
@@ -40,7 +40,8 @@
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 <pre><code><span class="keyword">fn</span> <span class="function declaration">not_static</span><span class="parenthesis">(</span><span class="parenthesis">)</span> <span class="brace">{</span><span class="brace">}</span>
 
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_attributes.html b/crates/ide/src/syntax_highlighting/test_data/highlight_attributes.html
@@ -40,7 +40,8 @@
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 <pre><code><span class="attribute_bracket attribute">#</span><span class="attribute_bracket attribute">[</span><span class="builtin_attr attribute library">allow</span><span class="parenthesis attribute">(</span><span class="none attribute">dead_code</span><span class="parenthesis attribute">)</span><span class="attribute_bracket attribute">]</span>
 <span class="attribute_bracket attribute">#</span><span class="attribute_bracket attribute">[</span><span class="tool_module attribute library">rustfmt</span><span class="operator attribute">::</span><span class="tool_module attribute library">skip</span><span class="attribute_bracket attribute">]</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_crate_root.html b/crates/ide/src/syntax_highlighting/test_data/highlight_crate_root.html
@@ -40,7 +40,8 @@
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 <pre><code><span class="keyword">extern</span> <span class="keyword">crate</span> <span class="module crate_root library">foo</span><span class="semicolon">;</span>
 <span class="keyword">use</span> <span class="module crate_root default_library library">core</span><span class="operator">::</span><span class="module default_library library">iter</span><span class="semicolon">;</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_default_library.html b/crates/ide/src/syntax_highlighting/test_data/highlight_default_library.html
@@ -40,7 +40,8 @@
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 <pre><code><span class="keyword">use</span> <span class="module crate_root default_library library">core</span><span class="operator">::</span><span class="module default_library library">iter</span><span class="semicolon">;</span>
 
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html b/crates/ide/src/syntax_highlighting/test_data/highlight_doctest.html
@@ -40,7 +40,8 @@
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 <pre><code><span class="comment documentation">//! This is a module to test doc injection.</span>
 <span class="comment documentation">//! ```</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_extern_crate.html b/crates/ide/src/syntax_highlighting/test_data/highlight_extern_crate.html
@@ -40,7 +40,8 @@
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 <pre><code><span class="keyword">extern</span> <span class="keyword">crate</span> <span class="module crate_root default_library library">std</span><span class="semicolon">;</span>
 <span class="keyword">extern</span> <span class="keyword">crate</span> <span class="module crate_root default_library library">alloc</span> <span class="keyword">as</span> <span class="module crate_root default_library declaration library">abc</span><span class="semicolon">;</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_general.html b/crates/ide/src/syntax_highlighting/test_data/highlight_general.html
@@ -40,7 +40,8 @@
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 <pre><code><span class="keyword">use</span> <span class="module">inner</span><span class="operator">::</span><span class="brace">{</span><span class="self_keyword">self</span> <span class="keyword">as</span> <span class="module declaration">inner_mod</span><span class="brace">}</span><span class="semicolon">;</span>
 <span class="keyword">mod</span> <span class="module declaration">inner</span> <span class="brace">{</span><span class="brace">}</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html b/crates/ide/src/syntax_highlighting/test_data/highlight_injection.html
@@ -40,7 +40,8 @@
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 <pre><code><span class="keyword">fn</span> <span class="function declaration">fixture</span><span class="parenthesis">(</span><span class="value_param declaration reference">ra_fixture</span><span class="colon">:</span> <span class="punctuation">&</span><span class="builtin_type">str</span><span class="parenthesis">)</span> <span class="brace">{</span><span class="brace">}</span>
 
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_keywords.html b/crates/ide/src/syntax_highlighting/test_data/highlight_keywords.html
@@ -40,7 +40,8 @@
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 <pre><code><span class="keyword">extern</span> <span class="keyword">crate</span> <span class="self_keyword crate_root public">self</span><span class="semicolon">;</span>
 
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_lifetimes.html b/crates/ide/src/syntax_highlighting/test_data/highlight_lifetimes.html
@@ -40,7 +40,8 @@
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 <pre><code>
 <span class="attribute_bracket attribute">#</span><span class="attribute_bracket attribute">[</span><span class="attribute attribute default_library library">derive</span><span class="parenthesis attribute">(</span><span class="parenthesis attribute">)</span><span class="attribute_bracket attribute">]</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_macros.html b/crates/ide/src/syntax_highlighting/test_data/highlight_macros.html
@@ -40,7 +40,8 @@
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 <pre><code><span class="module crate_root library">proc_macros</span><span class="operator">::</span><span class="macro library">mirror</span><span class="macro_bang">!</span> <span class="brace macro">{</span>
     <span class="brace macro">{</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_inline.html b/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_inline.html
@@ -40,7 +40,8 @@
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 <pre><code><span class="comment documentation">//! </span><span class="struct documentation injected intra_doc_link">[Struct]</span>
 <span class="comment documentation">//! This is an intra doc injection test for modules</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_outline.html b/crates/ide/src/syntax_highlighting/test_data/highlight_module_docs_outline.html
@@ -40,7 +40,8 @@
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 <pre><code><span class="comment documentation">/// </span><span class="struct documentation injected intra_doc_link">[crate::foo::Struct]</span>
 <span class="comment documentation">/// This is an intra doc injection test for modules</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_operators.html b/crates/ide/src/syntax_highlighting/test_data/highlight_operators.html
@@ -40,7 +40,8 @@
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 <pre><code><span class="keyword">fn</span> <span class="function declaration">main</span><span class="parenthesis">(</span><span class="parenthesis">)</span> <span class="brace">{</span>
     <span class="numeric_literal">1</span> <span class="arithmetic">+</span> <span class="numeric_literal">1</span> <span class="arithmetic">-</span> <span class="numeric_literal">1</span> <span class="arithmetic">*</span> <span class="numeric_literal">1</span> <span class="arithmetic">/</span> <span class="numeric_literal">1</span> <span class="arithmetic">%</span> <span class="numeric_literal">1</span> <span class="bitwise">|</span> <span class="numeric_literal">1</span> <span class="bitwise">&</span> <span class="numeric_literal">1</span> <span class="logical">!</span> <span class="numeric_literal">1</span> <span class="bitwise">^</span> <span class="numeric_literal">1</span> <span class="bitwise">&gt;&gt;</span> <span class="numeric_literal">1</span> <span class="bitwise">&lt;&lt;</span> <span class="numeric_literal">1</span><span class="semicolon">;</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_rainbow.html b/crates/ide/src/syntax_highlighting/test_data/highlight_rainbow.html
@@ -40,7 +40,8 @@
 .control            { font-style: italic; }
 .reference          { font-style: italic; font-weight: bold; }
 
-.unresolved_reference { color: #FC5555; text-decoration: wavy underline; }
+.invalid_escape_sequence { color: #FC5555; text-decoration: wavy underline; }
+.unresolved_reference    { color: #FC5555; text-decoration: wavy underline; }
 </style>
 <pre><code><span class="keyword">fn</span> <span class="function declaration">main</span><span class="parenthesis">(</span><span class="parenthesis">)</span> <span class="brace">{</span>
     <span class="keyword">let</span> <span class="variable declaration reference" data-binding-hash="8121853618659664005" style="color: hsl(273,88%,88%);">hello</span> <span class="operator">=</span> <span class="string_literal">"hello"</span><span class="semicolon">;</span>
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html b/crates/ide/src/syntax_highlighting/test_data/highlight_strings.html
diff --git a/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html b/crates/ide/src/syntax_highlighting/test_data/highlight_unsafe.html
diff --git a/crates/ide/src/syntax_highlighting/tests.rs b/crates/ide/src/syntax_highlighting/tests.rs
diff --git a/crates/rust-analyzer/src/semantic_tokens.rs b/crates/rust-analyzer/src/semantic_tokens.rs
diff --git a/crates/rust-analyzer/src/to_proto.rs b/crates/rust-analyzer/src/to_proto.rs
diff --git a/crates/syntax/src/ast/token_ext.rs b/crates/syntax/src/ast/token_ext.rs
diff --git a/editors/code/package.json b/editors/code/package.json

Original file line number	Diff line number	Diff line change
`@@ -1355,7 +1355,6 @@ impl<'ctx> MirLowerCtx<'ctx> {`
`1355`	`1355`	`return Ok(Operand::from_concrete_const(data, mm, ty));`
`1356`	`1356`	`}`
`1357`	`1357`	`hir_def::hir::Literal::CString(b) => {`
`1358`		`- let b = b.as_bytes();`
`1359`	`1358`	`let bytes = b.iter().copied().chain(iter::once(0)).collect::<Vec<_>>();`
`1360`	`1359`
`1361`	`1360`	`let mut data = Vec::with_capacity(mem::size_of::<usize>() * 2);`