Skip to content

Commit b948d81

Browse files
committed
Auto merge of #25219 - Eljay:fix-comment-parsing, r=alexcrichton
Fixes #25182, parser didn't account for \r\n in regular comments, only doc-comments.
2 parents 9bebe5f + 2dcc200 commit b948d81

File tree

1 file changed

+39
-24
lines changed
  • src/libsyntax/parse/lexer

1 file changed

+39
-24
lines changed

src/libsyntax/parse/lexer/mod.rs

Lines changed: 39 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -403,45 +403,51 @@ impl<'a> StringReader<'a> {
403403
Some('/') => {
404404
self.bump();
405405
self.bump();
406+
406407
// line comments starting with "///" or "//!" are doc-comments
407-
if self.curr_is('/') || self.curr_is('!') {
408-
let start_bpos = self.pos - BytePos(3);
409-
while !self.is_eof() {
410-
match self.curr.unwrap() {
411-
'\n' => break,
412-
'\r' => {
413-
if self.nextch_is('\n') {
414-
// CRLF
415-
break
416-
} else {
417-
self.err_span_(self.last_pos, self.pos,
418-
"bare CR not allowed in doc-comment");
419-
}
408+
let doc_comment = self.curr_is('/') || self.curr_is('!');
409+
let start_bpos = if doc_comment {
410+
self.pos - BytePos(3)
411+
} else {
412+
self.last_pos - BytePos(2)
413+
};
414+
415+
while !self.is_eof() {
416+
match self.curr.unwrap() {
417+
'\n' => break,
418+
'\r' => {
419+
if self.nextch_is('\n') {
420+
// CRLF
421+
break
422+
} else if doc_comment {
423+
self.err_span_(self.last_pos, self.pos,
424+
"bare CR not allowed in doc-comment");
420425
}
421-
_ => ()
422426
}
423-
self.bump();
427+
_ => ()
424428
}
425-
return self.with_str_from(start_bpos, |string| {
426-
// but comments with only more "/"s are not
429+
self.bump();
430+
}
431+
432+
return if doc_comment {
433+
self.with_str_from(start_bpos, |string| {
434+
// comments with only more "/"s are not doc comments
427435
let tok = if is_doc_comment(string) {
428436
token::DocComment(token::intern(string))
429437
} else {
430438
token::Comment
431439
};
432440

433-
return Some(TokenAndSpan{
441+
Some(TokenAndSpan {
434442
tok: tok,
435443
sp: codemap::mk_sp(start_bpos, self.last_pos)
436-
});
437-
});
444+
})
445+
})
438446
} else {
439-
let start_bpos = self.last_pos - BytePos(2);
440-
while !self.curr_is('\n') && !self.is_eof() { self.bump(); }
441-
return Some(TokenAndSpan {
447+
Some(TokenAndSpan {
442448
tok: token::Comment,
443449
sp: codemap::mk_sp(start_bpos, self.last_pos)
444-
});
450+
})
445451
}
446452
}
447453
Some('*') => {
@@ -1563,4 +1569,13 @@ mod tests {
15631569
assert_eq!(lexer.next_token().tok, token::Literal(token::Char(token::intern("a")), None));
15641570
}
15651571

1572+
#[test] fn crlf_comments() {
1573+
let sh = mk_sh();
1574+
let mut lexer = setup(&sh, "// test\r\n/// test\r\n".to_string());
1575+
let comment = lexer.next_token();
1576+
assert_eq!(comment.tok, token::Comment);
1577+
assert_eq!(comment.sp, ::codemap::mk_sp(BytePos(0), BytePos(7)));
1578+
assert_eq!(lexer.next_token().tok, token::Whitespace);
1579+
assert_eq!(lexer.next_token().tok, token::DocComment(token::intern("/// test")));
1580+
}
15661581
}

0 commit comments

Comments
 (0)