[flang][preprocessing] Mix preprocessing directives with free form li… (#96244)

…ne continuation

Allow preprocessing directives to appear between a source line and its
continuation, including conditional compilation directives (#if, #ifdef,
&c.).

Fixes #95476.

Author: klausler

flang/include/flang/Parser/preprocessor.h

@@ -82,6 +82,7 @@ class Preprocessor {
   bool IsNameDefined(const CharBlock &);
   bool IsFunctionLikeDefinition(const CharBlock &);
   bool AnyDefinitions() const { return !definitions_.empty(); }
+  bool InConditional() const { return !ifStack_.empty(); }
 
   // When called with partialFunctionLikeMacro not null, MacroReplacement()
   // and ReplaceMacros() handle an unclosed function-like macro reference

flang/lib/Parser/prescan.cpp

@@ -105,13 +105,15 @@ void Prescanner::Statement() {
     NextLine();
     return;
   case LineClassification::Kind::ConditionalCompilationDirective:
-  case LineClassification::Kind::DefinitionDirective:
-  case LineClassification::Kind::PreprocessorDirective:
+  case LineClassification::Kind::IncludeDirective:
     preprocessor_.Directive(TokenizePreprocessorDirective(), *this);
+    afterPreprocessingDirective_ = true;
+    skipLeadingAmpersand_ |= !inFixedForm_;
     return;
-  case LineClassification::Kind::IncludeDirective:
+  case LineClassification::Kind::PreprocessorDirective:
+  case LineClassification::Kind::DefinitionDirective:
     preprocessor_.Directive(TokenizePreprocessorDirective(), *this);
-    afterIncludeDirective_ = true;
+    // Don't set afterPreprocessingDirective_
     return;
   case LineClassification::Kind::CompilerDirective: {
     directiveSentinel_ = line.sentinel;
@@ -171,15 +173,17 @@ void Prescanner::Statement() {
         NextChar();
       }
       LabelField(tokens);
-    } else if (skipLeadingAmpersand_) {
-      skipLeadingAmpersand_ = false;
-      const char *p{SkipWhiteSpace(at_)};
-      if (p < limit_ && *p == '&') {
-        column_ += ++p - at_;
-        at_ = p;
-      }
     } else {
-      SkipSpaces();
+      if (skipLeadingAmpersand_) {
+        skipLeadingAmpersand_ = false;
+        const char *p{SkipWhiteSpace(at_)};
+        if (p < limit_ && *p == '&') {
+          column_ += ++p - at_;
+          at_ = p;
+        }
+      } else {
+        SkipSpaces();
+      }
       // Check for a leading identifier that might be a keyword macro
       // that will expand to anything indicating a non-source line, like
       // a comment marker or directive sentinel.  If so, disable line
@@ -289,13 +293,14 @@ void Prescanner::CheckAndEmitLine(
   tokens.CheckBadFortranCharacters(
       messages_, *this, disableSourceContinuation_);
   // Parenthesis nesting check does not apply while any #include is
-  // active, nor on the lines before and after a top-level #include.
+  // active, nor on the lines before and after a top-level #include,
+  // nor before or after conditional source.
   // Applications play shenanigans with line continuation before and
-  // after #include'd subprogram argument lists.
+  // after #include'd subprogram argument lists and conditional source.
   if (!isNestedInIncludeDirective_ && !omitNewline_ &&
-      !afterIncludeDirective_ && tokens.BadlyNestedParentheses()) {
-    if (inFixedForm_ && nextLine_ < limit_ &&
-        IsPreprocessorDirectiveLine(nextLine_)) {
+      !afterPreprocessingDirective_ && tokens.BadlyNestedParentheses() &&
+      !preprocessor_.InConditional()) {
+    if (nextLine_ < limit_ && IsPreprocessorDirectiveLine(nextLine_)) {
       // don't complain
     } else {
       tokens.CheckBadParentheses(messages_);
@@ -306,7 +311,7 @@ void Prescanner::CheckAndEmitLine(
     omitNewline_ = false;
   } else {
     cooked_.Put('\n', newlineProvenance);
-    afterIncludeDirective_ = false;
+    afterPreprocessingDirective_ = false;
   }
 }
 
@@ -353,10 +358,11 @@ void Prescanner::LabelField(TokenSequence &token) {
     ++column_;
   }
   if (badColumn && !preprocessor_.IsNameDefined(token.CurrentOpenToken())) {
-    if (prescannerNesting_ > 0 && *badColumn == 6 &&
-        cooked_.BufferedBytes() == firstCookedCharacterOffset_) {
-      // This is the first source line in #included text or conditional
-      // code under #if.
+    if ((prescannerNesting_ > 0 && *badColumn == 6 &&
+            cooked_.BufferedBytes() == firstCookedCharacterOffset_) ||
+        afterPreprocessingDirective_) {
+      // This is the first source line in #include'd text or conditional
+      // code under #if, or the first source line after such.
       // If it turns out that the preprocessed text begins with a
       // fixed form continuation line, the newline at the end
       // of the latest source line beforehand will be deleted in
@@ -599,7 +605,7 @@ bool Prescanner::NextToken(TokenSequence &tokens) {
       char previous{at_ <= start_ ? ' ' : at_[-1]};
       NextChar();
       SkipSpaces();
-      if (*at_ == '\n') {
+      if (*at_ == '\n' && !omitNewline_) {
         // Discard white space at the end of a line.
       } else if (!inPreprocessorDirective_ &&
           (previous == '(' || *at_ == '(' || *at_ == ')')) {
@@ -1069,6 +1075,17 @@ bool Prescanner::SkipCommentLine(bool afterAmpersand) {
     return true;
   } else if (inPreprocessorDirective_) {
     return false;
+  } else if (afterAmpersand &&
+      (lineClass.kind ==
+              LineClassification::Kind::ConditionalCompilationDirective ||
+          lineClass.kind == LineClassification::Kind::DefinitionDirective ||
+          lineClass.kind == LineClassification::Kind::PreprocessorDirective ||
+          lineClass.kind == LineClassification::Kind::IncludeDirective ||
+          lineClass.kind == LineClassification::Kind::IncludeLine)) {
+    SkipToEndOfLine();
+    omitNewline_ = true;
+    skipLeadingAmpersand_ = true;
+    return false;
   } else if (lineClass.kind ==
           LineClassification::Kind::ConditionalCompilationDirective ||
       lineClass.kind == LineClassification::Kind::PreprocessorDirective) {
@@ -1080,13 +1097,6 @@ bool Prescanner::SkipCommentLine(bool afterAmpersand) {
     // continued line).
     preprocessor_.Directive(TokenizePreprocessorDirective(), *this);
     return true;
-  } else if (afterAmpersand &&
-      (lineClass.kind == LineClassification::Kind::IncludeDirective ||
-          lineClass.kind == LineClassification::Kind::IncludeLine)) {
-    SkipToEndOfLine();
-    omitNewline_ = true;
-    skipLeadingAmpersand_ = true;
-    return false;
   } else {
     return false;
   }

flang/lib/Parser/prescan.h

@@ -214,7 +214,7 @@ class Prescanner {
   int prescannerNesting_{0};
   int continuationLines_{0};
   bool isPossibleMacroCall_{false};
-  bool afterIncludeDirective_{false};
+  bool afterPreprocessingDirective_{false};
   bool disableSourceContinuation_{false};
 
   Provenance startProvenance_;

flang/test/Preprocessing/cond-contin.F90

@@ -0,0 +1,21 @@
+! RUN: %flang_fc1 -E %s 2>&1 | FileCheck %s
+! CHECK: subroutine test(ARG1, FA, FB, ARG2)
+! CHECK: end
+
+subroutine test( &
+ARG1, &
+! test
+#ifndef SWAP
+#define ARG1 FA
+#define ARG2 FB
+#else
+#define ARG1 FB
+#define ARG2 FA
+#endif
+ARG1, ARG2, &
+! test
+#undef ARG1
+#undef ARG2
+&ARG2)
+! comment
+end

flang/test/Preprocessing/inc-contin-1.F

@@ -0,0 +1,6 @@
+! RUN: %flang_fc1 -E %s 2>&1 | FileCheck %s
+! CHECK: call t(1  ,.false.)
+      program main
+#include "inc-contin-1.h"
+     $,.false.)
+      end

flang/test/Preprocessing/inc-contin-1.h

@@ -0,0 +1 @@
+      call t(1

flang/test/Preprocessing/inc-contin-2.F90

@@ -0,0 +1,9 @@
+! RUN: %flang_fc1 -E %s 2>&1 | FileCheck %s
+! CHECK: print *, 3.14159
+! CHECK: print *, 3. 14159
+      program main
+#include "inc-contin-2a.h"
+     &14159
+#include "inc-contin-2b.h"
+     &14159
+      end program main

flang/test/Preprocessing/inc-contin-2a.h

@@ -0,0 +1 @@
+print *, 3.&

flang/test/Preprocessing/inc-contin-2b.h

@@ -0,0 +1 @@
+print *, 3. &

flang/test/Preprocessing/include-args.F90

@@ -1,5 +1,5 @@
 ! RUN: %flang -E %s 2>&1 | FileCheck %s
-! CHECK: call foo(3.14159)
+! CHECK: call foo(3.14159 )
 call foo (&
 #include "args.h"
 )

flang/unittests/Frontend/FrontendActionTest.cpp

@@ -143,7 +143,7 @@ TEST_F(FrontendActionTest, PrintPreprocessedInput) {
   EXPECT_TRUE(success);
   EXPECT_TRUE(!outputFileBuffer.empty());
   EXPECT_TRUE(
-      llvm::StringRef(outputFileBuffer.data()).starts_with("program b\n"));
+      llvm::StringRef(outputFileBuffer.data()).starts_with(" program b\n"));
 }
 
 TEST_F(FrontendActionTest, ParseSyntaxOnly) {