Implement a new type of FileID: FileIDInfo::MacroExpansion. For tokens that

came from a macro expansion, this allows us to keep track of both where the
character data came from and where the logical position of the token is (at
the expansion site).  This implements Preprocessor/indent_macro.c, and
reduces the number of cpp iostream -E diffs vs GCC from 2589 to 2297.

llvm-svn: 38557

Author: lattner

clang/Basic/SourceManager.cpp

@@ -110,6 +110,16 @@ unsigned SourceManager::createFileID(const InfoRec *File,
   return Result;
 }
 
+/// createFileIDForMacroExp - Return a new FileID for a macro expansion at
+/// SourcePos, where the macro token character came from PhysicalFileID.
+///
+unsigned SourceManager::createFileIDForMacroExp(SourceLocation SourcePos, 
+                                                unsigned PhysicalFileID) {
+  FileIDs.push_back(FileIDInfo::getMacroExpansion(SourcePos, PhysicalFileID));
+  return FileIDs.size();
+}
+
+
 /// getCharacterData - Return a pointer to the start of the specified location
 /// in the appropriate SourceBuffer.  This returns null if it cannot be
 /// computed (e.g. invalid SourceLocation).
@@ -126,9 +136,16 @@ const char *SourceManager::getCharacterData(SourceLocation SL) const {
 unsigned SourceManager::getColumnNumber(SourceLocation IncludePos) const {
   unsigned FileID = IncludePos.getFileID();
   if (FileID == 0) return 0;
-  FileInfo *FileInfo = getFileInfo(FileID);
+  
+  // If this is a macro, we need to get the instantiation location.
+  const SrcMgr::FileIDInfo *FIDInfo = getFIDInfo(FileID);
+  if (FIDInfo->IDType == SrcMgr::FileIDInfo::MacroExpansion) {
+    IncludePos = FIDInfo->IncludeLoc;
+    FileID = IncludePos.getFileID();
+  }
+  
   unsigned FilePos = getFilePos(IncludePos);
-  const SourceBuffer *Buffer = FileInfo->Buffer;
+  const SourceBuffer *Buffer = getBuffer(FileID);
   const char *Buf = Buffer->getBufferStart();
 
   unsigned LineStart = FilePos;
@@ -211,6 +228,19 @@ void SourceManager::PrintStats() const {
   std::cerr << FileInfos.size() << " files mapped, " << MemBufferInfos.size()
             << " mem buffers mapped, " << FileIDs.size() 
             << " file ID's allocated.\n";
+  unsigned NumBuffers = 0, NumMacros = 0;
+  for (unsigned i = 0, e = FileIDs.size(); i != e; ++i) {
+    if (FileIDs[i].IDType == FileIDInfo::NormalBuffer)
+      ++NumBuffers;
+    else if (FileIDs[i].IDType == FileIDInfo::MacroExpansion)
+      ++NumMacros;
+    else
+      assert(0 && "Unknown FileID!");
+  }
+  std::cerr << "  " << NumBuffers << " normal buffer FileID's, "
+            << NumMacros << " macro expansion FileID's.\n";
+    
+  
 
   unsigned NumLineNumsComputed = 0;
   unsigned NumFileBytesMapped = 0;

clang/Lex/MacroExpander.cpp

@@ -14,6 +14,7 @@
 #include "clang/Lex/MacroExpander.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Basic/SourceManager.h"
 using namespace llvm;
 using namespace clang;
 
@@ -24,9 +25,6 @@ MacroExpander::MacroExpander(LexerToken &Tok, Preprocessor &pp)
     HasLeadingSpace(Tok.hasLeadingSpace()) {
 }
 
-
-
-
 /// Lex - Lex and return a token from this macro stream.
 ///
 void MacroExpander::Lex(LexerToken &Tok) {
@@ -38,6 +36,18 @@ void MacroExpander::Lex(LexerToken &Tok) {
   Tok = Macro.getReplacementToken(CurToken++);
   //Tok.SetLocation(InstantiateLoc);
 
+  // The token's current location indicate where the token was lexed from.  We
+  // need this information to compute the spelling of the token, but any
+  // diagnostics for the expanded token should appear as if they came from
+  // InstantiateLoc.  Pull this information together into a new SourceLocation
+  // that captures all of this.
+  unsigned CharFilePos = Tok.getLocation().getRawFilePos();
+  unsigned CharFileID  = Tok.getLocation().getFileID();
+  
+  unsigned InstantiationFileID =
+    PP.getSourceManager().createFileIDForMacroExp(InstantiateLoc, CharFileID);
+  Tok.SetLocation(SourceLocation(InstantiationFileID, CharFilePos));
+
   // If this is the first token, set the lexical properties of the token to
   // match the lexical properties of the macro identifier.
   if (CurToken == 1) {

clang/Lex/Preprocessor.cpp

@@ -349,8 +349,6 @@ void Preprocessor::EnterSourceFile(unsigned FileID,
 void Preprocessor::EnterMacro(LexerToken &Tok) {
   IdentifierTokenInfo *Identifier = Tok.getIdentifierInfo();
   MacroInfo &MI = *Identifier->getMacroInfo();
-  SourceLocation ExpandLoc = Tok.getLocation();
-  //unsigned MacroID = SourceMgr.getMacroID(Identifier, ExpandLoc);
   if (CurLexer) {
     IncludeStack.push_back(IncludeStackInfo(CurLexer, CurNextDirLookup));
     CurLexer         = 0;
@@ -367,7 +365,6 @@ void Preprocessor::EnterMacro(LexerToken &Tok) {
   // Mark the macro as currently disabled, so that it is not recursively
   // expanded.
   MI.DisableMacro();
-  
   CurMacroExpander = new MacroExpander(Tok, *this);
 }
 

clang/include/clang/Basic/SourceManager.h

@@ -67,7 +67,9 @@ namespace SrcMgr {
   ///      the SourceLocation's offset from the start of the buffer.
   ///   2. Macro Expansions.  These indicate that the logical location is
   ///      totally different than the physical location.  The logical source
-  ///      location is specified with an explicit SourceLocation object.
+  ///      location is specified by the IncludeLoc.  The physical location is
+  ///      the FilePos of the token's SourceLocation combined with the FileID
+  ///      from MacroTokenFileID.
   ///
   struct FileIDInfo {
     enum FileIDType {
@@ -76,7 +78,7 @@ namespace SrcMgr {
     };
 
     /// The type of this FileID.
-    FileIDType IDType : 2;
+    FileIDType IDType;
 
     /// IncludeLoc - The location of the #include that brought in this file.
     /// This SourceLocation object has a FileId of 0 for the main file.
@@ -96,9 +98,9 @@ namespace SrcMgr {
         const InfoRec *Info;
       } NormalBuffer;
 
-      /// MacroTokenLoc - This is the raw encoding of a SourceLocation which
-      /// indicates the physical location of the macro token.
-      unsigned MacroTokenLoc;
+      /// MacroTokenFileID - This is the File ID that contains the characters
+      /// that make up the expanded token.
+      unsigned MacroTokenFileID;
     } u;
 
     /// getNormalBuffer - Return a FileIDInfo object for a normal buffer
@@ -114,13 +116,14 @@ namespace SrcMgr {
     }
 
     /// getMacroExpansion - Return a FileID for a macro expansion.  IL specifies
-    /// the instantiation location, 
+    /// the instantiation location, and MacroFID specifies the FileID that the
+    /// token's characters come from. 
     static FileIDInfo getMacroExpansion(SourceLocation IL,
-                                        SourceLocation TokenLoc) {
+                                        unsigned MacroFID) {
       FileIDInfo X;
       X.IDType = MacroExpansion;
       X.IncludeLoc = IL;
-      X.u.MacroTokenLoc = TokenLoc.getRawEncoding();
+      X.u.MacroTokenFileID = MacroFID;
       return X;
     }
 
@@ -172,10 +175,15 @@ class SourceManager {
     return createFileID(createMemBufferInfoRec(Buffer), SourceLocation());
   }
 
+  /// createFileIDForMacroExp - Return a new FileID for a macro expansion at
+  /// SourcePos, where the macro token character came from PhysicalFileID.
+  ///
+  unsigned createFileIDForMacroExp(SourceLocation SourcePos, 
+                                   unsigned PhysicalFileID);
 
   /// getBuffer - Return the buffer for the specified FileID.
   ///
-  const SourceBuffer *getBuffer(unsigned FileID) {
+  const SourceBuffer *getBuffer(unsigned FileID) const {
     return getFileInfo(FileID)->Buffer;
   }
 
@@ -187,12 +195,18 @@ class SourceManager {
   }
 
   /// getFilePos - This (efficient) method returns the offset from the start of
-  /// the file that the specified SourceLocation represents.
+  /// the file that the specified SourceLocation represents.  This returns the
+  /// location of the physical character data, not the logical file position.
   unsigned getFilePos(SourceLocation IncludePos) const {
-    assert(IncludePos.getFileID()-1 < FileIDs.size() && "Invalid FileID!");
+    const SrcMgr::FileIDInfo *FIDInfo = getFIDInfo(IncludePos.getFileID());
+
+    // For Macros, the physical loc is specified by the MacroTokenFileID.
+    if (FIDInfo->IDType == SrcMgr::FileIDInfo::MacroExpansion)
+      FIDInfo = &FileIDs[FIDInfo->u.MacroTokenFileID-1];
+    
     // If this file has been split up into chunks, factor in the chunk number
     // that the FileID references.
-    unsigned ChunkNo=FileIDs[IncludePos.getFileID()-1].getNormalBufferChunkNo();
+    unsigned ChunkNo = FIDInfo->getNormalBufferChunkNo();
     return IncludePos.getRawFilePos() +
            (ChunkNo << SourceLocation::FilePosBits);
   }
@@ -237,9 +251,21 @@ class SourceManager {
   /// buffer.  This does no caching.
   const SrcMgr::InfoRec *createMemBufferInfoRec(const SourceBuffer *Buffer);
 
-  const SrcMgr::InfoRec *getInfoRec(unsigned FileID) const {
+  const SrcMgr::FileIDInfo *getFIDInfo(unsigned FileID) const {
     assert(FileID-1 < FileIDs.size() && "Invalid FileID!");
-    return FileIDs[FileID-1].getNormalBufferInfo();
+    return &FileIDs[FileID-1];
+  }
+    
+  /// Return the InfoRec structure for the specified FileID.  This is always the
+  /// physical reference for the ID.
+  const SrcMgr::InfoRec *getInfoRec(unsigned FileID) const {
+    const SrcMgr::FileIDInfo *FIDInfo = getFIDInfo(FileID);
+
+    // For Macros, the physical loc is specified by the MacroTokenFileID.
+    if (FIDInfo->IDType == SrcMgr::FileIDInfo::MacroExpansion)
+      FIDInfo = &FileIDs[FIDInfo->u.MacroTokenFileID-1];
+    
+    return FIDInfo->getNormalBufferInfo();
   }
 
   SrcMgr::FileInfo *getFileInfo(unsigned FileID) const {