analyzer work on issues with flex-generated lexers [PR103546]

PR analyzer/103546 tracks various false positives seen on
flex-generated lexers.

Whilst investigating them, I noticed an ICE with
-fanalyzer-call-summaries due to attempting to store sm-state
for an UNKNOWN svalue, which this patch fixes.

This patch also provides known_function implementations of all of the
external functions called by the lexer, reducing the number of false
positives.

The patch doesn't eliminate all false positives, but adds integration
tests to try to establish a baseline from which the remaining false
positives can be fixed.

gcc/analyzer/ChangeLog:
	PR analyzer/103546
	* analyzer.h (register_known_file_functions): New decl.
	* program-state.cc (sm_state_map::replay_call_summary): Rejct
	attempts to store sm-state for caller_sval that can't have
	associated state.
	* region-model-impl-calls.cc (register_known_functions): Call
	register_known_file_functions.
	* sm-fd.cc (class kf_isatty): New.
	(register_known_fd_functions): Register it.
	* sm-file.cc (class kf_ferror): New.
	(class kf_fileno): New.
	(class kf_getc): New.
	(register_known_file_functions): New.

gcc/ChangeLog:
	PR analyzer/103546
	* doc/invoke.texi (Static Analyzer Options): Add isatty, ferror,
	fileno, and getc to the list of functions known to the analyzer.

gcc/testsuite/ChangeLog:
	PR analyzer/103546
	* gcc.dg/analyzer/ferror-1.c: New test.
	* gcc.dg/analyzer/fileno-1.c: New test.
	* gcc.dg/analyzer/flex-with-call-summaries.c: New test.
	* gcc.dg/analyzer/flex-without-call-summaries.c: New test.
	* gcc.dg/analyzer/getc-1.c: New test.
	* gcc.dg/analyzer/isatty-1.c: New test.

Signed-off-by: David Malcolm <dmalcolm@redhat.com>

5 files changed, 136 insertions, 0 deletions

diff --git a/gcc/analyzer/analyzer.h b/gcc/analyzer/analyzer.h
index 4fbe092..35c71f3 100644
--- a/gcc/analyzer/analyzer.h
+++ b/gcc/analyzer/analyzer.h
@@ -259,6 +259,7 @@ public:
 
 extern void register_known_functions (known_function_manager &mgr);
 extern void register_known_fd_functions (known_function_manager &kfm);
+extern void register_known_file_functions (known_function_manager &kfm);
 extern void register_varargs_builtins (known_function_manager &kfm);
 
 /* Passed by pointer to PLUGIN_ANALYZER_INIT callbacks.  */
diff --git a/gcc/analyzer/program-state.cc b/gcc/analyzer/program-state.cc
index 037dbec..3942b5f 100644
--- a/gcc/analyzer/program-state.cc
+++ b/gcc/analyzer/program-state.cc
@@ -821,6 +821,8 @@ sm_state_map::replay_call_summary (call_summary_replay &r,
       const svalue *caller_sval = r.convert_svalue_from_summary (summary_sval);
       if (!caller_sval)
 	continue;
+      if (!caller_sval->can_have_associated_state_p ())
+	continue;
       const svalue *summary_origin = kv.second.m_origin;
       const svalue *caller_origin
 	= (summary_origin
diff --git a/gcc/analyzer/region-model-impl-calls.cc b/gcc/analyzer/region-model-impl-calls.cc
index 37cb09f..6d8c9f9 100644
--- a/gcc/analyzer/region-model-impl-calls.cc
+++ b/gcc/analyzer/region-model-impl-calls.cc
@@ -1662,6 +1662,7 @@ register_known_functions (known_function_manager &kfm)
     kfm.add ("putenv", make_unique<kf_putenv> ());
 
     register_known_fd_functions (kfm);
+    register_known_file_functions (kfm);
   }
 
   /* glibc functions.  */
diff --git a/gcc/analyzer/sm-fd.cc b/gcc/analyzer/sm-fd.cc
index 8f8ec85..794733e 100644
--- a/gcc/analyzer/sm-fd.cc
+++ b/gcc/analyzer/sm-fd.cc
@@ -2487,6 +2487,84 @@ public:
   }
 };
 
+/* Handler for "isatty"".
+   See e.g. https://man7.org/linux/man-pages/man3/isatty.3.html  */
+
+class kf_isatty : public known_function
+{
+  class outcome_of_isatty : public succeed_or_fail_call_info
+  {
+  public:
+    outcome_of_isatty (const call_details &cd, bool success)
+    : succeed_or_fail_call_info (cd, success)
+    {}
+
+    bool update_model (region_model *model,
+		       const exploded_edge *,
+		       region_model_context *ctxt) const final override
+    {
+      const call_details cd (get_call_details (model, ctxt));
+
+      if (m_success)
+	{
+	  /* Return 1.  */
+	  model->update_for_int_cst_return (cd, 1, true);
+	}
+      else
+	{
+	  /* Return 0; set errno.  */
+	  model->update_for_int_cst_return (cd, 0, true);
+	  model->set_errno (cd);
+	}
+
+      return feasible_p (cd, ctxt);
+    }
+
+  private:
+    bool feasible_p (const call_details &cd,
+		     region_model_context *ctxt) const
+    {
+      if (m_success)
+	{
+	  /* Can't be "success" on a closed/invalid fd.  */
+	  sm_state_map *smap;
+	  const fd_state_machine *fd_sm;
+	  std::unique_ptr<sm_context> sm_ctxt;
+	  if (!get_fd_state (ctxt, &smap, &fd_sm, NULL, &sm_ctxt))
+	    return true;
+	  const extrinsic_state *ext_state = ctxt->get_ext_state ();
+	  if (!ext_state)
+	    return true;
+
+	  const svalue *fd_sval = cd.get_arg_svalue (0);
+	  state_machine::state_t old_state
+	    = sm_ctxt->get_state (cd.get_call_stmt (), fd_sval);
+
+	  if (fd_sm->is_closed_fd_p (old_state)
+	      || old_state == fd_sm->m_invalid)
+	    return false;
+	}
+      return true;
+    }
+  }; // class outcome_of_isatty
+
+public:
+  bool matches_call_types_p (const call_details &cd) const final override
+  {
+    return cd.num_args () == 1;
+  }
+
+  void impl_call_post (const call_details &cd) const final override
+  {
+    if (cd.get_ctxt ())
+      {
+	cd.get_ctxt ()->bifurcate (make_unique<outcome_of_isatty> (cd, false));
+	cd.get_ctxt ()->bifurcate (make_unique<outcome_of_isatty> (cd, true));
+	cd.get_ctxt ()->terminate_path ();
+      }
+  }
+};
+
 /* Handler for calls to "pipe" and "pipe2".
    See e.g. https://www.man7.org/linux/man-pages/man2/pipe.2.html  */
 
@@ -2582,6 +2660,7 @@ register_known_fd_functions (known_function_manager &kfm)
   kfm.add ("accept", make_unique<kf_accept> ());
   kfm.add ("bind", make_unique<kf_bind> ());
   kfm.add ("connect", make_unique<kf_connect> ());
+  kfm.add ("isatty", make_unique<kf_isatty> ());
   kfm.add ("listen", make_unique<kf_listen> ());
   kfm.add ("pipe", make_unique<kf_pipe> (1));
   kfm.add ("pipe2", make_unique<kf_pipe> (2));
diff --git a/gcc/analyzer/sm-file.cc b/gcc/analyzer/sm-file.cc
index cbd1788..083c0ec 100644
--- a/gcc/analyzer/sm-file.cc
+++ b/gcc/analyzer/sm-file.cc
@@ -489,6 +489,59 @@ make_fileptr_state_machine (logger *logger)
   return new fileptr_state_machine (logger);
 }
 
+/* Handler for "ferror"".  */
+
+class kf_ferror : public known_function
+{
+public:
+  bool matches_call_types_p (const call_details &cd) const final override
+  {
+    return (cd.num_args () == 1
+	    && cd.arg_is_pointer_p (0));
+  }
+
+  /* No side effects.  */
+};
+
+/* Handler for "fileno"".  */
+
+class kf_fileno : public known_function
+{
+public:
+  bool matches_call_types_p (const call_details &cd) const final override
+  {
+    return (cd.num_args () == 1
+	    && cd.arg_is_pointer_p (0));
+  }
+
+  /* No side effects.  */
+};
+
+/* Handler for "getc"".  */
+
+class kf_getc : public known_function
+{
+public:
+  bool matches_call_types_p (const call_details &cd) const final override
+  {
+    return (cd.num_args () == 1
+	    && cd.arg_is_pointer_p (0));
+  }
+
+  /* No side effects.  */
+};
+
+/* Populate KFM with instances of known functions relating to
+   stdio streams.  */
+
+void
+register_known_file_functions (known_function_manager &kfm)
+{
+  kfm.add ("ferror", make_unique<kf_ferror> ());
+  kfm.add ("fileno", make_unique<kf_fileno> ());
+  kfm.add ("getc", make_unique<kf_getc> ());
+}
+
 #if CHECKING_P
 
 namespace selftest {