[flang] Allow compiler directives in macros

Modify the prescanner to allow compiler directives to appear in
macro expansions, and adjust the parser to accept a semicolon
as a directive terminator.

Differential Revision: https://reviews.llvm.org/D150780

1 files changed, 31 insertions, 4 deletions

diff --git a/flang/lib/Parser/token-sequence.cpp b/flang/lib/Parser/token-sequence.cpp
index a3b97d3..eaa2bf3 100644
--- a/flang/lib/Parser/token-sequence.cpp
+++ b/flang/lib/Parser/token-sequence.cpp
@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "token-sequence.h"
+#include "prescan.h"
 #include "flang/Parser/characters.h"
 #include "flang/Parser/message.h"
 #include "llvm/Support/raw_ostream.h"
@@ -244,11 +245,31 @@ TokenSequence &TokenSequence::RemoveRedundantBlanks(std::size_t firstChar) {
   return *this;
 }
 
-TokenSequence &TokenSequence::ClipComment(bool skipFirst) {
+TokenSequence &TokenSequence::ClipComment(
+    const Prescanner &prescanner, bool skipFirst) {
   std::size_t tokens{SizeInTokens()};
   for (std::size_t j{0}; j < tokens; ++j) {
-    if (TokenAt(j).FirstNonBlank() == '!') {
-      if (skipFirst) {
+    CharBlock tok{TokenAt(j)};
+    if (std::size_t blanks{tok.CountLeadingBlanks()};
+        blanks < tok.size() && tok[blanks] == '!') {
+      // Retain active compiler directive sentinels (e.g. "!dir$")
+      for (std::size_t k{j + 1}; k < tokens && tok.size() < blanks + 5; ++k) {
+        if (tok.begin() + tok.size() == TokenAt(k).begin()) {
+          tok.ExtendToCover(TokenAt(k));
+        } else {
+          break;
+        }
+      }
+      bool isSentinel{false};
+      if (tok.size() == blanks + 5) {
+        char sentinel[4];
+        for (int k{0}; k < 4; ++k) {
+          sentinel[k] = ToLowerCaseLetter(tok[blanks + k + 1]);
+        }
+        isSentinel = prescanner.IsCompilerDirectiveSentinel(sentinel, 4);
+      }
+      if (isSentinel) {
+      } else if (skipFirst) {
         skipFirst = false;
       } else {
         TokenSequence result;
@@ -315,11 +336,12 @@ ProvenanceRange TokenSequence::GetProvenanceRange() const {
 const TokenSequence &TokenSequence::CheckBadFortranCharacters(
     Messages &messages) const {
   std::size_t tokens{SizeInTokens()};
+  bool isBangOk{true};
   for (std::size_t j{0}; j < tokens; ++j) {
     CharBlock token{TokenAt(j)};
     char ch{token.FirstNonBlank()};
     if (ch != ' ' && !IsValidFortranTokenCharacter(ch)) {
-      if (ch == '!' && j == 0) {
+      if (ch == '!' && isBangOk) {
         // allow in !dir$
       } else if (ch < ' ' || ch >= '\x7f') {
         messages.Say(GetTokenProvenanceRange(j),
@@ -329,6 +351,11 @@ const TokenSequence &TokenSequence::CheckBadFortranCharacters(
             "bad character ('%c') in Fortran token"_err_en_US, ch);
       }
     }
+    if (ch == ';') {
+      isBangOk = true;
+    } else if (ch != ' ') {
+      isBangOk = false;
+    }
   }
   return *this;
 }