automatic import of gccopeneuler22.03_LTS

1 files changed, 1000 insertions, 0 deletions

diff --git a/0027-Autoprefetch-Support-auto-feedback-prefetch.patch b/0027-Autoprefetch-Support-auto-feedback-prefetch.patch
new file mode 100644
index 0000000..c3dcf50
--- /dev/null
+++ b/0027-Autoprefetch-Support-auto-feedback-prefetch.patch
@@ -0,0 +1,1000 @@
+From 6b944bed1158d3454b1db27aeab4ec1f2b8e5866 Mon Sep 17 00:00:00 2001
+From: huangxiaoquan <huangxiaoquan1@huawei.com>
+Date: Thu, 27 Jan 2022 18:24:53 +0800
+Subject: [PATCH 27/28] [Autoprefetch] Support auto feedback prefetch
+
+1.Add option -fprefetch-loop-arrays=[value].
+
+2.A prefetch distance analysis algorithm based on branch weight
+  is proposed to improve the accuracy of prefetch distance.
+
+3.Propose automatic feedback prefetching:
+  use the cache-miss profile information to guide the insertion of
+  prefetching instructions.
+---
+ gcc/auto-profile.c           |   5 +-
+ gcc/common.opt               |   5 +
+ gcc/opts.c                   |   7 +
+ gcc/params.opt               |  16 +
+ gcc/tree-ssa-loop-prefetch.c | 735 ++++++++++++++++++++++++++++++++++-
+ 5 files changed, 748 insertions(+), 20 deletions(-)
+
+diff --git a/gcc/auto-profile.c b/gcc/auto-profile.c
+index e6164b91b..f221978fc 100644
+--- a/gcc/auto-profile.c
++++ b/gcc/auto-profile.c
+@@ -21,6 +21,8 @@ along with GCC; see the file COPYING3.  If not see
+ #include "config.h"
+ #define INCLUDE_MAP
+ #define INCLUDE_SET
++#define INCLUDE_ALGORITHM
++#define INCLUDE_VECTOR
+ #include "system.h"
+ #include "coretypes.h"
+ #include "backend.h"
+@@ -49,9 +51,6 @@ along with GCC; see the file COPYING3.  If not see
+ #include "auto-profile.h"
+ #include "tree-pretty-print.h"
+ #include "gimple-pretty-print.h"
+-#include <map>
+-#include <vector>
+-#include <algorithm>
+ 
+ /* The following routines implements AutoFDO optimization.
+ 
+diff --git a/gcc/common.opt b/gcc/common.opt
+index 37cbbd8c0..9488bd90f 100644
+--- a/gcc/common.opt
++++ b/gcc/common.opt
+@@ -2201,6 +2201,11 @@ fprefetch-loop-arrays
+ Common Report Var(flag_prefetch_loop_arrays) Init(-1) Optimization
+ Generate prefetch instructions, if available, for arrays in loops.
+ 
++fprefetch-loop-arrays=
++Common Joined RejectNegative UInteger Var(prefetch_level) Init(0) IntegerRange(0, 3)
++Generate prefetch instructions, if available, for arrays in loops. The prefetch
++level can control the optimize level to array prefetch.
++
+ fprofile
+ Common Report Var(profile_flag)
+ Enable basic program profiling code.
+diff --git a/gcc/opts.c b/gcc/opts.c
+index 7a39f618b..f49f5ee58 100644
+--- a/gcc/opts.c
++++ b/gcc/opts.c
+@@ -1747,6 +1747,8 @@ set_cache_misses_profile_params (struct gcc_options *opts,
+ 			       struct gcc_options *opts_set)
+ {
+   SET_OPTION_IF_UNSET (opts, opts_set, flag_prefetch_loop_arrays, 1);
++  SET_OPTION_IF_UNSET (opts, opts_set, prefetch_level, 2);
++  SET_OPTION_IF_UNSET (opts, opts_set, param_simultaneous_prefetches, 100);
+ }
+ 
+ /* -f{,no-}sanitize{,-recover}= suboptions.  */
+@@ -2645,6 +2647,11 @@ common_handle_option (struct gcc_options *opts,
+       SET_OPTION_IF_UNSET (opts, opts_set, flag_ipa_bit_cp, value);
+       break;
+ 
++    case OPT_fprefetch_loop_arrays_:
++      opts->x_prefetch_level = value;
++      opts->x_flag_prefetch_loop_arrays = true;
++      break;
++
+     case OPT_fpatchable_function_entry_:
+       {
+ 	char *patch_area_arg = xstrdup (arg);
+diff --git a/gcc/params.opt b/gcc/params.opt
+index 2db69cc87..9d1faa7ab 100644
+--- a/gcc/params.opt
++++ b/gcc/params.opt
+@@ -968,4 +968,20 @@ Bound on number of runtime checks inserted by the vectorizer's loop versioning f
+ Common Joined UInteger Var(param_vect_max_version_for_alignment_checks) Init(6) Param Optimization
+ Bound on number of runtime checks inserted by the vectorizer's loop versioning for alignment check.
+ 
++-param=param-prefetch-func-topn=
++Common Joined UInteger Var(param_prefetch_func_topn) Init(3) Param Optimization
++TopN functions of cache miss counts to be analyzed in prefetching.
++
++-param=param-prefetch-ref-topn=
++Common Joined UInteger Var(param_prefetch_ref_topn) Init(5) Param Optimization
++TopN ref of cache miss counts to be analyzed in prefetching.
++
++-param=param-high-loop-execution-rate=
++Common Joined UInteger Var(param_high_loop_execution_rate) Init(95) IntegerRange(0, 100) Param Optimization
++High execution rate loops to be analyzed in prefetch (in%).
++
++-param=param-prefetch-func-counts-threshold=
++Common Joined UInteger Var(param_prefetch_func_counts_threshold) Init(100) Param Optimization
++Threshold functions of cache miss counts to be analyzed in prefetching.
++
+ ; This comment is to ensure we retain the blank line above.
+diff --git a/gcc/tree-ssa-loop-prefetch.c b/gcc/tree-ssa-loop-prefetch.c
+index d19ece641..3a5aef0fc 100644
+--- a/gcc/tree-ssa-loop-prefetch.c
++++ b/gcc/tree-ssa-loop-prefetch.c
+@@ -18,6 +18,9 @@ along with GCC; see the file COPYING3.  If not see
+ <http://www.gnu.org/licenses/>.  */
+ 
+ #include "config.h"
++#define INCLUDE_ALGORITHM
++#define INCLUDE_MAP
++#define INCLUDE_VECTOR
+ #include "system.h"
+ #include "coretypes.h"
+ #include "backend.h"
+@@ -48,6 +51,11 @@ along with GCC; see the file COPYING3.  If not see
+ #include "tree-data-ref.h"
+ #include "diagnostic-core.h"
+ #include "dbgcnt.h"
++#include "gimple-pretty-print.h"
++#include "tree-cfg.h"
++#include "auto-profile.h"
++#include "cgraph.h"
++#include "print-tree.h"
+ 
+ /* This pass inserts prefetch instructions to optimize cache usage during
+    accesses to arrays in loops.  It processes loops sequentially and:
+@@ -253,6 +261,22 @@ struct mem_ref_group
+ #define PREFETCH_MAX_MEM_REFS_PER_LOOP 200
+ #endif
+ 
++#ifndef PREFETCH_FUNC_TOPN
++#define PREFETCH_FUNC_TOPN param_prefetch_func_topn
++#endif
++
++#ifndef PREFETCH_FUNC_COUNTS_THRESHOLD
++#define PREFETCH_FUNC_COUNTS_THRESHOLD param_prefetch_func_counts_threshold
++#endif
++
++#ifndef PREFETCH_REF_TOPN
++#define PREFETCH_REF_TOPN param_prefetch_ref_topn
++#endif
++
++#ifndef LOOP_EXECUTION_RATE
++#define LOOP_EXECUTION_RATE param_high_loop_execution_rate
++#endif
++
+ /* The memory reference.  */
+ 
+ struct mem_ref
+@@ -279,6 +303,131 @@ struct mem_ref
+ 				   nontemporal one.  */
+ };
+ 
++/* Probability information of basic blocks and branches.  */
++struct bb_bp
++{
++  basic_block bb;
++  basic_block true_edge_bb;
++  basic_block false_edge_bb;
++  float true_edge_prob;
++  float false_edge_prob;
++  float bb_prob;
++};
++
++typedef struct bb_bp bb_bp;
++
++enum PREFETCH_MODE
++{
++  ORIGINAL_MODE=0,		/* Original prefetch method.  */
++  REFINE_BB_AHEAD,
++				/* Prefetch distance algorithm for removing
++				   irrelevant bb.  */
++  BRANCH_WEIGHTED_AHEAD,
++				/* Branch weighted prefetch
++				   distance algorithm.  */
++  INDIRECT_MODE			/* Indirect array prefetch mode.  */
++};
++
++typedef std::map <unsigned int, unsigned int> uid_rank_map;
++typedef std::map <location_t, unsigned int> loc_rank_map;
++typedef std::vector <std::pair<location_t, gcov_type> > loc_gcov_type_vec;
++typedef std::map <location_t, std::vector<gimple *> > loc_gimple_vec_map;
++
++static loc_rank_map ref_rank;
++
++/* Callback function for event_count comparison.  */
++
++static bool
++event_count_cmp (std::pair<unsigned int, gcov_type> &a,
++		 std::pair<unsigned int, gcov_type> &b)
++{
++  return a.second > b.second;
++}
++
++/* Prepared mappings from location to counts and from location
++   to stmt list.  */
++
++static void
++prepare_loc_count_info (function *fun, loc_gcov_type_vec &ref_sorted,
++			loc_gimple_vec_map &loc_stmt, event_type event)
++{
++  basic_block bb = NULL;
++  gimple_stmt_iterator bsi;
++  gimple *stmt;
++  tree lhs = NULL_TREE;
++  tree rhs = NULL_TREE;
++
++  FOR_EACH_BB_FN (bb, fun)
++    {
++      for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
++	{
++	  stmt = gsi_stmt (bsi);
++	  if (gimple_code (stmt) != GIMPLE_ASSIGN)
++	    {
++	      continue;
++	    }
++	  if (!gimple_vuse (stmt))
++	   {
++	      continue;
++	   }
++	  lhs = gimple_assign_lhs (stmt);
++	  rhs = gimple_assign_rhs1 (stmt);
++	  if (REFERENCE_CLASS_P (rhs) || REFERENCE_CLASS_P (lhs))
++	    {
++	      gcov_type loc_count =
++	        event_get_loc_count (gimple_location (stmt), event);
++	      if (loc_count > 0)
++		{
++		  /* There may be multiple gimple correspond to the same
++		     location.  */
++		  if (loc_stmt.count (gimple_location (stmt)) == 0)
++		    {
++		      ref_sorted.push_back (std::make_pair (gimple_location (stmt),
++					loc_count));
++		    }
++		  loc_stmt[gimple_location (stmt)].push_back (stmt);
++		}
++	    }
++	}
++    }
++}
++
++/* Sort references by event_count and dump loc count information after 
++   sorting.  */
++
++static void
++sort_ref_by_event_count (function *fun, event_type event)
++{
++  loc_gcov_type_vec ref_sorted;
++  loc_gimple_vec_map loc_stmt;
++
++  prepare_loc_count_info (fun, ref_sorted, loc_stmt, event);
++  sort (ref_sorted.begin (), ref_sorted.end (), event_count_cmp);
++
++  for (unsigned i = 0; i < ref_sorted.size (); ++i)
++    {
++      ref_rank[ref_sorted[i].first] = i + 1;
++      /* Print the stmt and count of the topn ref.  */
++      if (i < PREFETCH_REF_TOPN && dump_file && (dump_flags & TDF_DETAILS))
++	{
++	  fprintf (dump_file, "stmt: \n");
++	  for (unsigned j = 0; j < loc_stmt[ref_sorted[i].first].size ();
++		++j)
++	    {
++	      print_gimple_stmt (dump_file,
++				 loc_stmt[ref_sorted[i].first][j], 0);
++	    }
++	  gcov_type loc_count =
++	    event_get_loc_count (ref_sorted[i].first, event);
++	  fprintf (dump_file, "stmt loc %u counts is %lu: "
++			      "rank %d in top %d, (candidate analysis)\n\n",
++		   ref_sorted[i].first, loc_count,
++		   ref_rank[ref_sorted[i].first], PREFETCH_REF_TOPN);
++	}
++    }
++  return;
++}
++
+ /* Dumps information about memory reference */
+ static void
+ dump_mem_details (FILE *file, tree base, tree step,
+@@ -479,6 +628,30 @@ idx_analyze_ref (tree base, tree *index, void *data)
+   return true;
+ }
+ 
++/* Dumps information about ar_data structure.  */
++
++static void
++dump_ar_data_details (FILE *file, tree ref, struct ar_data &ar_data)
++{
++  print_generic_expr (file, ref, TDF_SLIM);
++  fprintf (file, "\n");
++  if (*(ar_data.step))
++    {
++      fprintf (file, " step ");
++      if (cst_and_fits_in_hwi (*(ar_data.step)))
++	fprintf (file, HOST_WIDE_INT_PRINT_DEC,
++		 int_cst_value (*(ar_data.step)));
++      else
++	print_generic_expr (file, *(ar_data.step), TDF_SLIM);
++    }
++  fprintf (file, "\n");
++  if (*(ar_data.delta))
++    {
++      fprintf (file, " delta " HOST_WIDE_INT_PRINT_DEC "\n",
++	       *(ar_data.delta));
++    }
++}
++
+ /* Tries to express REF_P in shape &BASE + STEP * iter + DELTA, where DELTA and
+    STEP are integer constants and iter is number of iterations of LOOP.  The
+    reference occurs in statement STMT.  Strips nonaddressable component
+@@ -526,7 +699,17 @@ analyze_ref (class loop *loop, tree *ref_p, tree *base,
+   ar_data.stmt = stmt;
+   ar_data.step = step;
+   ar_data.delta = delta;
+-  return for_each_index (base, idx_analyze_ref, &ar_data);
++
++  if (dump_file && (dump_flags & TDF_DETAILS))
++    {
++      dump_ar_data_details (dump_file, ref, ar_data);
++    }
++  bool idx_flag = for_each_index (base, idx_analyze_ref, &ar_data);
++  if (dump_file && (dump_flags & TDF_DETAILS))
++    {
++      fprintf (dump_file, "idx_flag = %d \n\n", idx_flag);
++    }
++  return idx_flag;
+ }
+ 
+ /* Record a memory reference REF to the list REFS.  The reference occurs in
+@@ -601,6 +784,55 @@ gather_memory_references_ref (class loop *loop, struct mem_ref_group **refs,
+   return true;
+ }
+ 
++/* Determine whether to collect the memory references based on the
++   ranking of ref cache miss counts.  */
++
++static bool
++should_gather_memory_references (gimple *stmt)
++{
++  if (!(profile_exist (CACHE_MISSES)))
++    {
++      return true;
++    }
++
++  if (dump_file && (dump_flags & TDF_DETAILS))
++    {
++      fprintf (dump_file, "stmt:");
++      print_gimple_stmt (dump_file, stmt, 0);
++      fprintf (dump_file, "\n");
++    }
++  if (ref_rank.count (gimple_location (stmt)) == 0)
++    {
++      if (dump_file && (dump_flags & TDF_DETAILS))
++	{
++	  fprintf (dump_file, "stmt location no found, skip prefetch "
++			      "analysis\n");
++	}
++      return false;
++    }
++  gcov_type loc_count = event_get_loc_count (gimple_location (stmt), CACHE_MISSES);
++  if (ref_rank[gimple_location (stmt)] > PREFETCH_REF_TOPN)
++    {
++      if (dump_file && (dump_flags & TDF_DETAILS))
++	{
++	  fprintf (dump_file, "stmt loc %u counts is %lu:"
++			      "rank %d exceed topn %d, skip prefetch "
++			      "analysis\n",
++		   gimple_location (stmt), loc_count,
++		   ref_rank[gimple_location (stmt)], PREFETCH_REF_TOPN);
++	}
++      return false;
++    }
++  if (dump_file && (dump_flags & TDF_DETAILS))
++    {
++      fprintf (dump_file, "stmt loc %u counts is %lu: rank %d in top %d,"
++			  "continue prefetch analysis\n",
++	       gimple_location (stmt), loc_count,
++	       ref_rank[gimple_location (stmt)], PREFETCH_REF_TOPN);
++    }
++  return true;
++}
++
+ /* Record the suitable memory references in LOOP.  NO_OTHER_REFS is set to
+    true if there are no other memory references inside the loop.  */
+ 
+@@ -626,6 +858,13 @@ gather_memory_references (class loop *loop, bool *no_other_refs, unsigned *ref_c
+       if (bb->loop_father != loop)
+ 	continue;
+ 
++      if (dump_file && (dump_flags & TDF_DETAILS))
++	{
++	  fprintf (dump_file, "==== the %dth loop bb body ====\n", i);
++	  gimple_dump_bb (dump_file, bb, 0, dump_flags);
++	  fprintf (dump_file, "\n");
++	}
++
+       for (bsi = gsi_start_bb (bb); !gsi_end_p (bsi); gsi_next (&bsi))
+ 	{
+ 	  stmt = gsi_stmt (bsi);
+@@ -642,20 +881,31 @@ gather_memory_references (class loop *loop, bool *no_other_refs, unsigned *ref_c
+ 	  if (! gimple_vuse (stmt))
+ 	    continue;
+ 
++	  if (!should_gather_memory_references (stmt))
++	    continue;
++
+ 	  lhs = gimple_assign_lhs (stmt);
+ 	  rhs = gimple_assign_rhs1 (stmt);
+ 
+ 	  if (REFERENCE_CLASS_P (rhs))
+ 	    {
+-	    *no_other_refs &= gather_memory_references_ref (loop, &refs,
+-							    rhs, false, stmt);
+-	    *ref_count += 1;
++	      if (dump_file && (dump_flags & TDF_DETAILS))
++		{
++		  fprintf (dump_file, "====> the %dth ref \n", *ref_count);
++		}
++	      *no_other_refs &= gather_memory_references_ref (loop, &refs, rhs,
++							      false, stmt);
++	      *ref_count += 1;
+ 	    }
+ 	  if (REFERENCE_CLASS_P (lhs))
+ 	    {
+-	    *no_other_refs &= gather_memory_references_ref (loop, &refs,
+-							    lhs, true, stmt);
+-	    *ref_count += 1;
++	      if (dump_file && (dump_flags & TDF_DETAILS))
++		{
++		  fprintf (dump_file, "====> the %dth ref \n", *ref_count);
++		}
++	      *no_other_refs &= gather_memory_references_ref (loop, &refs, lhs,
++							      true, stmt);
++	      *ref_count += 1;
+ 	    }
+ 	}
+     }
+@@ -1168,9 +1418,9 @@ issue_prefetch_ref (struct mem_ref *ref, unsigned unroll_factor, unsigned ahead)
+   bool nontemporal = ref->reuse_distance >= L2_CACHE_SIZE_BYTES;
+ 
+   if (dump_file && (dump_flags & TDF_DETAILS))
+-    fprintf (dump_file, "Issued%s prefetch for reference %u:%u.\n",
+-	     nontemporal ? " nontemporal" : "",
+-	     ref->group->uid, ref->uid);
++      fprintf (dump_file, "Issued%s prefetch for reference %u:%u.\n",
++	       nontemporal ? " nontemporal" : "",
++	       ref->group->uid, ref->uid);
+ 
+   bsi = gsi_for_stmt (ref->stmt);
+ 
+@@ -1875,6 +2125,306 @@ insn_to_prefetch_ratio_too_small_p (unsigned ninsns, unsigned prefetch_count,
+   return false;
+ }
+ 
++/* Obtain the edge probability information of each basic block in the loop.  */
++
++static float
++get_edge_prob (edge e)
++{
++  /* Limit the minimum probability value.  */
++  const float MINNUM_PROB = 0.00001f;
++  float fvalue = 1;
++
++  profile_probability probability = e->probability;
++  if (probability.initialized_p ())
++    {
++      fvalue = probability.to_reg_br_prob_base () / float (REG_BR_PROB_BASE);
++      if (fvalue < MINNUM_PROB && probability.to_reg_br_prob_base ())
++	{
++	  fvalue = MINNUM_PROB;
++	}
++    }
++  return fvalue;
++}
++
++
++/* Dump the bb information in a loop.  */
++
++static void
++dump_loop_bb (struct loop *loop)
++{
++  basic_block *body = get_loop_body_in_dom_order (loop);
++  basic_block bb = NULL;
++
++  for (unsigned i = 0; i < loop->num_nodes; i++)
++    {
++      bb = body[i];
++      if (bb->loop_father != loop)
++	{
++	  continue;
++	}
++      if (dump_file && (dump_flags & TDF_DETAILS))
++	{
++	  fprintf (dump_file, "===== the %dth loop bb body ======= \n", i);
++	  gimple_dump_bb (dump_file, bb, 0, dump_flags);
++	  fprintf (dump_file, "\n");
++	}
++    }
++  free (body);
++}
++
++
++/* Obtain the branch probability information of each basic block
++   in the loop.  */
++
++static void
++get_bb_branch_prob (hash_map <basic_block, bb_bp> &bb_branch_prob,
++		 struct loop *loop)
++{
++  basic_block *body = get_loop_body (loop);
++  basic_block bb = NULL;
++  for (unsigned i = 0; i < loop->num_nodes; i++)
++    {
++      bb = body[i];
++      if (bb->loop_father != loop)
++	{
++	  continue;
++	}
++      bb_bp &branch_prob = bb_branch_prob.get_or_insert (bb);
++      branch_prob.bb = bb;
++      branch_prob.true_edge_bb = NULL;
++      branch_prob.false_edge_bb = NULL;
++      branch_prob.true_edge_prob = 0;
++      branch_prob.false_edge_prob = 0;
++      branch_prob.bb_prob = 0;
++
++      gimple *stmt = last_stmt (bb);
++      if (stmt && gimple_code (stmt) == GIMPLE_COND)
++	{
++	  if (EDGE_COUNT (bb->succs) != 2)
++	    {
++	      if (dump_file && (dump_flags & TDF_DETAILS))
++		fprintf (dump_file, "The number of successful edges of bb"
++				    "is abnormal\n");
++	      continue;
++	    }
++	  edge true_edge = NULL;
++	  edge false_edge = NULL;
++	  extract_true_false_edges_from_block (bb, &true_edge, &false_edge);
++
++	  /* If it is exiting bb, and the destination bb of the edge does not
++	     belong to the current loop, the information of the edge is not
++	     recorded.  */
++	  if (true_edge->dest->loop_father == loop)
++	    {
++	      branch_prob.true_edge_bb = true_edge->dest;
++	      branch_prob.true_edge_prob = get_edge_prob (true_edge);
++	    }
++	  if (false_edge->dest->loop_father == loop)
++	    {
++	      branch_prob.false_edge_bb = false_edge->dest;
++	      branch_prob.false_edge_prob = get_edge_prob (false_edge);
++	    }
++	}
++
++      edge e = find_fallthru_edge (bb->succs);
++      if (e)
++	{
++	  branch_prob.true_edge_bb = e->dest;
++	  branch_prob.true_edge_prob = get_edge_prob (e);
++	}
++    }
++}
++
++/* Traverse each bb in the loop and prune fake loops.  */
++
++static bool
++traverse_prune_bb_branch (hash_map <basic_block, bb_bp> &bb_branch_prob,
++			  int& max_path, hash_set <basic_block> &path_node,
++			  basic_block current_bb, basic_block latch_bb)
++{
++  /* Limit the maximum number of analysis paths.  */
++  if (max_path <= 0 || current_bb == NULL)
++    return false;
++
++  /* Do not join edges that do not form a complete loop.  */
++  bb_bp *bb_bp_node = bb_branch_prob.get (current_bb);
++  if (bb_bp_node == NULL || (bb_bp_node->true_edge_bb == NULL
++			     && bb_bp_node->false_edge_bb == NULL))
++    return false;
++
++  if (current_bb == latch_bb)
++    {
++      max_path--;
++      return true;
++    }
++
++  /* Do not join edges that return to non-dominate nodes.  */
++  if (path_node.contains (bb_bp_node->true_edge_bb)
++      || path_node.contains (bb_bp_node->false_edge_bb))
++    {
++      if (dump_file && (dump_flags & TDF_DETAILS))
++	fprintf (dump_file, "fake loop: in bb%d\n", current_bb->index);
++      return false;
++    }
++
++  path_node.add (current_bb);
++  if (bb_bp_node->true_edge_bb)
++    {
++      if (traverse_prune_bb_branch (bb_branch_prob, max_path,
++		path_node, bb_bp_node->true_edge_bb, latch_bb) == false)
++	return false;
++    }
++  if (bb_bp_node->false_edge_bb)
++    {
++      if (traverse_prune_bb_branch (bb_branch_prob, max_path,
++		path_node, bb_bp_node->false_edge_bb, latch_bb) == false)
++	return false;
++    }
++  path_node.remove (current_bb);
++
++  max_path--;
++  return true;
++}
++
++/* Traverse and calculate the probability of basic block.  */
++
++static void
++traverse_calculate_bb_prob (hash_map <basic_block, bb_bp> &bb_branch_prob,
++			    basic_block current_bb, basic_block latch_bb,
++			    float prob)
++{
++  /* Limit bb block access probability, the probability is
++     less than 100% and include delta.  */
++  const float MAX_BB_PROBABILITY = 1.001f;
++
++  if (current_bb == NULL)
++    {
++      return;
++    }
++  bb_bp *bb_bp_node = bb_branch_prob.get (current_bb);
++  bb_bp_node->bb_prob += prob;
++
++  gcc_assert (bb_bp_node->bb_prob <= MAX_BB_PROBABILITY);
++
++  if (bb_bp_node == NULL || (bb_bp_node->true_edge_bb == NULL
++			     && bb_bp_node->false_edge_bb == NULL))
++    {
++      return;
++    }
++  if (current_bb == latch_bb)
++    {
++      return;
++    }
++
++  bool assign = (bb_bp_node->true_edge_bb && bb_bp_node->false_edge_bb);
++  if (bb_bp_node->true_edge_bb)
++    {
++      float assign_prob = assign ? bb_bp_node->true_edge_prob * prob : prob;
++      traverse_calculate_bb_prob (bb_branch_prob,
++			    bb_bp_node->true_edge_bb, latch_bb, assign_prob);
++    }
++  if (bb_bp_node->false_edge_bb)
++    {
++      float assign_prob = assign ? bb_bp_node->false_edge_prob * prob : prob;
++      traverse_calculate_bb_prob (bb_branch_prob,
++			    bb_bp_node->false_edge_bb, latch_bb, assign_prob);
++    }
++  return;
++}
++
++/* Obtain the probability of basic block.  */
++
++static bool
++get_bb_prob (hash_map <basic_block, bb_bp> &bb_branch_prob, struct loop *loop)
++{
++  /* The upper limit of the branch path in the loop is 10000.  */
++  const int MAX_BB_BRANCH_PATH = 10000;
++
++  if (loop->header == NULL || loop->latch == NULL
++      || loop->header == loop->latch)
++    {
++      if (dump_file && (dump_flags & TDF_DETAILS))
++	fprintf (dump_file, "get_bb_prob failed: without the header bb or "
++			    "latch bb\n");
++      return false;
++    }
++
++  bb_bp *latch_branch_prob = bb_branch_prob.get (loop->latch);
++  bb_bp *header_branch_prob = bb_branch_prob.get (loop->header);
++  if (header_branch_prob == NULL || latch_branch_prob == NULL
++      || (latch_branch_prob->true_edge_bb != header_branch_prob->bb
++	  && latch_branch_prob->false_edge_bb != header_branch_prob->bb))
++    {
++      if (dump_file && (dump_flags & TDF_DETAILS))
++	fprintf (dump_file, "get_bb_prob failed: loop data exception\n");
++      return false;
++    }
++
++  hash_set <basic_block> path_node;
++  int max_path = MAX_BB_BRANCH_PATH;
++  if (traverse_prune_bb_branch (bb_branch_prob, max_path, path_node,
++			    header_branch_prob->bb, loop->latch) == false)
++    {
++      if (dump_file && (dump_flags & TDF_DETAILS))
++	fprintf (dump_file, "traverse_prune_bb_branch false.\n");
++      return false;
++    }
++  traverse_calculate_bb_prob (bb_branch_prob,
++			      header_branch_prob->bb, loop->latch, 1);
++
++  return true;
++}
++
++/* Computes an estimated number of insns in LOOP, weighted by WEIGHTS.  */
++
++static unsigned
++estimate_num_loop_insns (struct loop *loop, eni_weights *weights)
++{
++  basic_block *body = get_loop_body_in_dom_order (loop);
++  gimple_stmt_iterator gsi;
++  float size = 0;
++  basic_block bb = NULL;
++  hash_map <basic_block, bb_bp> bb_branch_prob;
++
++  if (prefetch_level >= BRANCH_WEIGHTED_AHEAD)
++    {
++      get_bb_branch_prob (bb_branch_prob, loop);
++      if (get_bb_prob (bb_branch_prob, loop) == false)
++	{
++	  dump_loop_bb (loop);
++	  return 0;
++	}
++    }
++
++  for (unsigned i = 0; i < loop->num_nodes; i++)
++    {
++      bb = body[i];
++      /* For nested loops, the bb of the inner loop is not calculated.  */
++      if (bb->loop_father != loop)
++	{
++	  continue;
++	}
++
++      float size_tmp = 0;
++      for (gsi = gsi_start_bb (body[i]); !gsi_end_p (gsi); gsi_next (&gsi))
++	{
++	  size_tmp += estimate_num_insns (gsi_stmt (gsi), weights);
++	}
++
++      if (prefetch_level >= BRANCH_WEIGHTED_AHEAD)
++	{
++	  float bb_prob = bb_branch_prob.get (bb)->bb_prob;
++	  size += size_tmp * bb_prob;
++	}
++      else
++	{
++	  size += size_tmp;
++	}
++    }
++  free (body);
++
++  return unsigned (size);
++}
+ 
+ /* Issue prefetch instructions for array references in LOOP.  Returns
+    true if the LOOP was unrolled.  */
+@@ -1899,7 +2449,15 @@ loop_prefetch_arrays (class loop *loop)
+ 
+   /* FIXME: the time should be weighted by the probabilities of the blocks in
+      the loop body.  */
+-  time = tree_num_loop_insns (loop, &eni_time_weights);
++
++  if (prefetch_level >= REFINE_BB_AHEAD)
++    {
++      time = estimate_num_loop_insns (loop, &eni_time_weights);
++    }
++  else
++    {
++      time = tree_num_loop_insns (loop, &eni_time_weights);
++    }
+   if (time == 0)
+     return false;
+ 
+@@ -1913,7 +2471,14 @@ loop_prefetch_arrays (class loop *loop)
+   if (trip_count_to_ahead_ratio_too_small_p (ahead, est_niter))
+     return false;
+ 
+-  ninsns = tree_num_loop_insns (loop, &eni_size_weights);
++  if (prefetch_level >= REFINE_BB_AHEAD)
++    {
++      ninsns = estimate_num_loop_insns (loop, &eni_size_weights);
++    }
++  else
++    {
++      ninsns = tree_num_loop_insns (loop, &eni_size_weights);
++    }
+ 
+   /* Step 1: gather the memory references.  */
+   refs = gather_memory_references (loop, &no_other_refs, &mem_ref_count);
+@@ -1978,10 +2543,49 @@ fail:
+   return unrolled;
+ }
+ 
++/* Determine if it is a high execution rate loop.  */
++
++static bool
++is_high_exec_rate_loop (struct loop *loop)
++{
++  vec<edge> exit_edges = get_loop_exit_edges (loop);
++  if (exit_edges == vNULL)
++    {
++      return false;
++    }
++
++  unsigned i = 0;
++  gcov_type exit_count = 0;
++  edge e = NULL;
++  float loop_exec_rate = 0;
++  gcov_type header_bb_count = loop->header->count.to_gcov_type ();
++  FOR_EACH_VEC_ELT (exit_edges, i, e)
++    {
++      gcov_type exiting_bb_count = e->src->count.to_gcov_type ();
++      float exit_edge_prob = get_edge_prob (e);
++      exit_count += exit_edge_prob * exiting_bb_count;
++
++      loop_exec_rate = 1.0 - ((double) exit_count / header_bb_count);
++
++      if (loop_exec_rate < (float) LOOP_EXECUTION_RATE / 100.0)
++	{
++	  return false;
++	}
++    }
++
++  if (dump_file && (dump_flags & TDF_DETAILS))
++    {
++      fprintf (dump_file, "loop with high execution rate: %f >= %f\n\n",
++	       loop_exec_rate, (float) LOOP_EXECUTION_RATE / 100.0);
++      dump_loop_bb (loop);
++    }
++  return true;
++}
++
+ /* Issue prefetch instructions for array references in loops.  */
+ 
+ unsigned int
+-tree_ssa_prefetch_arrays (void)
++tree_ssa_prefetch_arrays (function *fun)
+ {
+   class loop *loop;
+   bool unrolled = false;
+@@ -2012,6 +2616,12 @@ tree_ssa_prefetch_arrays (void)
+ 	       param_min_insn_to_prefetch_ratio);
+       fprintf (dump_file, "    min insn-to-mem ratio: %d \n",
+ 	       param_prefetch_min_insn_to_mem_ratio);
++      fprintf (dump_file, "    prefetch_func_topn: %d \n",
++	       param_prefetch_func_topn);
++      fprintf (dump_file, "    prefetch_ref_topn: %d \n",
++	       param_prefetch_ref_topn);
++      fprintf (dump_file, "    high_loop_execution_rate: %d \n",
++	       LOOP_EXECUTION_RATE);
+       fprintf (dump_file, "\n");
+     }
+ 
+@@ -2028,13 +2638,42 @@ tree_ssa_prefetch_arrays (void)
+       set_builtin_decl (BUILT_IN_PREFETCH, decl, false);
+     }
+ 
+-  FOR_EACH_LOOP (loop, LI_FROM_INNERMOST)
++  enum li_flags LI = LI_FROM_INNERMOST;
++
++  if (profile_exist (CACHE_MISSES))
++    {
++      LI = LI_ONLY_INNERMOST;
++    }
++  if (dump_file && (dump_flags & TDF_DETAILS))
++    {
++      fprintf (dump_file, "Processing model %d:\n", LI);
++    }
++
++  if (profile_exist (CACHE_MISSES))
++    {
++      sort_ref_by_event_count (fun, CACHE_MISSES);
++    }
++
++  FOR_EACH_LOOP (loop, LI)
+     {
+       if (dump_file && (dump_flags & TDF_DETAILS))
+-	fprintf (dump_file, "Processing loop %d:\n", loop->num);
++	{
++	  fprintf (dump_file, "======================================\n");
++	  fprintf (dump_file, "Processing loop %d:\n", loop->num);
++	  fprintf (dump_file, "======================================\n");
++	  flow_loop_dump (loop, dump_file, NULL, 1);
++	  fprintf (dump_file, "\n\n");
++	}
+ 
+-      unrolled |= loop_prefetch_arrays (loop);
++      if (profile_exist (CACHE_MISSES))
++	{
++	  if (!is_high_exec_rate_loop (loop))
++	    {
++	      continue;
++	    }
++	}
+ 
++      unrolled |= loop_prefetch_arrays (loop);
+       if (dump_file && (dump_flags & TDF_DETAILS))
+ 	fprintf (dump_file, "\n\n");
+     }
+@@ -2049,6 +2688,56 @@ tree_ssa_prefetch_arrays (void)
+   return todo_flags;
+ }
+ 
++/* Determine whether to analyze the function according to
++   the sorting of the function containing cache-miss counts.  */
++
++static bool
++should_analyze_func_p (void)
++{
++  gcov_type decl_uid = DECL_UID (current_function_decl);
++  struct rank_info func_rank_info =
++    event_get_func_rank (decl_uid, CACHE_MISSES);
++  if (func_rank_info.total == 0)
++    {
++      return false;
++    }
++  gcov_type func_count = event_get_func_count (decl_uid, CACHE_MISSES);
++  if (func_count == 0)
++    {
++      if (dump_file && (dump_flags & TDF_DETAILS))
++	{
++	  fprintf (dump_file, "function uid %d cannot find profile data "
++			      "and skip prefetch analysis\n",
++		   decl_uid);
++	}
++      return false;
++    }
++  if (func_rank_info.rank > PREFETCH_FUNC_TOPN
++      || func_count < PREFETCH_FUNC_COUNTS_THRESHOLD)
++    {
++      if (dump_file && (dump_flags & TDF_DETAILS))
++	{
++	  fprintf (dump_file, "function uid %d total counts is %lu: "
++			      "rank %d > topn %d, counts %lu < threshold %lu "
++			      "skip prefetch analysis\n",
++		   decl_uid, func_count,
++		   func_rank_info.rank, PREFETCH_FUNC_TOPN,
++		   func_count, PREFETCH_FUNC_COUNTS_THRESHOLD);
++	}
++      return false;
++    }
++  if (dump_file && (dump_flags & TDF_DETAILS))
++    {
++      fprintf (dump_file, "function uid %d total counts is %lu: "
++			  "rank %d in topn %d, counts %lu > threshold %lu "
++			  "continue prefetch analysis\n",
++	       decl_uid, func_count,
++	       func_rank_info.rank, PREFETCH_FUNC_TOPN,
++	       func_count, PREFETCH_FUNC_COUNTS_THRESHOLD);
++    }
++  return true;
++}
++
+ /* Prefetching.  */
+ 
+ namespace {
+@@ -2085,6 +2774,18 @@ pass_loop_prefetch::execute (function *fun)
+   if (number_of_loops (fun) <= 1)
+     return 0;
+ 
++  /* Filter only when combined with cache-miss. When the should_analyze_func_p
++     analysis fails (for example, the function without cache-miss count),
++     in order to ensure the accuracy of the prefetch analysis, the function
++     does not perform native prefetch processing.  */
++  if (profile_exist (CACHE_MISSES))
++    {
++      if (!should_analyze_func_p ())
++	{
++	  return 0;
++	}
++    }
++
+   if ((PREFETCH_BLOCK & (PREFETCH_BLOCK - 1)) != 0)
+     {
+       static bool warned = false;
+@@ -2099,7 +2800,7 @@ pass_loop_prefetch::execute (function *fun)
+       return 0;
+     }
+ 
+-  return tree_ssa_prefetch_arrays ();
++  return tree_ssa_prefetch_arrays (fun);
+ }
+ 
+ } // anon namespace
+-- 
+2.27.0.windows.1
+