auto merge of #14630 : cmr/rust/rewrite-lexer, r=alexcrichton

These are a pain to rebase, so I'm separating this from the rest of my work.
Nothing controversial here, just some simple refactoring and removal of an
unused entry in the token table. Brings the lexer into 2012 with methods!

Author: bors

src/librustdoc/html/highlight.rs

@@ -34,7 +34,7 @@ pub fn highlight(src: &str, class: Option<&str>) -> String {
 
     let mut out = io::MemWriter::new();
     doit(&sess,
-         lexer::new_string_reader(&sess.span_diagnostic, fm),
+         lexer::StringReader::new(&sess.span_diagnostic, fm),
          class,
          &mut out).unwrap();
     str::from_utf8_lossy(out.unwrap().as_slice()).to_string()

src/libsyntax/attr.rs

@@ -15,7 +15,7 @@ use ast::{AttrId, Attribute, Attribute_, MetaItem, MetaWord, MetaNameValue, Meta
 use codemap::{Span, Spanned, spanned, dummy_spanned};
 use codemap::BytePos;
 use diagnostic::SpanHandler;
-use parse::comments::{doc_comment_style, strip_doc_comment_decoration};
+use parse::lexer::comments::{doc_comment_style, strip_doc_comment_decoration};
 use parse::token::InternedString;
 use parse::token;
 use crateid::CrateId;