path: root/0164-Backport-SME-aarch64-Add-group-suffixes-to-SVE-intri.patch

From e99332e15895156632949f3b6c3080fc9d994b13 Mon Sep 17 00:00:00 2001
From: Richard Sandiford <richard.sandiford@arm.com>
Date: Tue, 5 Dec 2023 10:11:19 +0000
Subject: [PATCH 065/157] [Backport][SME] aarch64: Add group suffixes to SVE
 intrinsics

Reference: https://gcc.gnu.org/git/?p=gcc.git;a=commit;h=7b607f197967e052d7d7e29f6b41eded18f8c65d

The SME2 ACLE adds a new "group" suffix component to the naming
convention for SVE intrinsics.  This is also used in the new tuple
forms of the svreinterpret intrinsics.

This patch adds support for group suffixes and defines the
x2, x3 and x4 suffixes that are needed for the svreinterprets.

gcc/
	* config/aarch64/aarch64-sve-builtins-shapes.cc (build_one): Take
	a group suffix index parameter.
	(build_32_64, build_all): Update accordingly.  Iterate over all
	group suffixes.
	* config/aarch64/aarch64-sve-builtins-sve2.cc (svqrshl_impl::fold)
	(svqshl_impl::fold, svrshl_impl::fold): Update function_instance
	constructors.
	* config/aarch64/aarch64-sve-builtins.cc (group_suffixes): New array.
	(groups_none): New constant.
	(function_groups): Initialize the groups field.
	(function_instance::hash): Hash the group index.
	(function_builder::get_name): Add the group suffix.
	(function_builder::add_overloaded_functions): Iterate over all
	group suffixes.
	(function_resolver::lookup_form): Take a group suffix parameter.
	(function_resolver::resolve_to): Likewise.
	* config/aarch64/aarch64-sve-builtins.def (DEF_SVE_GROUP_SUFFIX): New
	macro.
	(x2, x3, x4): New group suffixes.
	* config/aarch64/aarch64-sve-builtins.h (group_suffix_index): New enum.
	(group_suffix_info): New structure.
	(function_group_info::groups): New member variable.
	(function_instance::group_suffix_id): Likewise.
	(group_suffixes): New array.
	(function_instance::operator==): Compare the group suffixes.
	(function_instance::group_suffix): New function.
---
 .../aarch64/aarch64-sve-builtins-shapes.cc    | 53 ++++++------
 .../aarch64/aarch64-sve-builtins-sve2.cc      | 10 +--
 gcc/config/aarch64/aarch64-sve-builtins.cc    | 84 +++++++++++++------
 gcc/config/aarch64/aarch64-sve-builtins.def   |  9 ++
 gcc/config/aarch64/aarch64-sve-builtins.h     | 81 ++++++++++++++----
 5 files changed, 165 insertions(+), 72 deletions(-)

diff --git a/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc b/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc
index 4fa4181b9..3ecef026c 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins-shapes.cc
@@ -275,18 +275,20 @@ parse_signature (const function_instance &instance, const char *format,
 }
 
 /* Add one function instance for GROUP, using mode suffix MODE_SUFFIX_ID,
-   the type suffixes at index TI and the predication suffix at index PI.
-   The other arguments are as for build_all.  */
+   the type suffixes at index TI, the group suffixes at index GI, and the
+   predication suffix at index PI.  The other arguments are as for
+   build_all.  */
 static void
 build_one (function_builder &b, const char *signature,
 	   const function_group_info &group, mode_suffix_index mode_suffix_id,
-	   unsigned int ti, unsigned int pi, bool force_direct_overloads)
+	   unsigned int ti, unsigned int gi, unsigned int pi,
+	   bool force_direct_overloads)
 {
   /* Byte forms of svdupq take 16 arguments.  */
   auto_vec<tree, 16> argument_types;
   function_instance instance (group.base_name, *group.base, *group.shape,
 			      mode_suffix_id, group.types[ti],
-			      group.preds[pi]);
+			      group.groups[gi], group.preds[pi]);
   tree return_type = parse_signature (instance, signature, argument_types);
   apply_predication (instance, return_type, argument_types);
   b.add_unique_function (instance, return_type, argument_types,
@@ -312,24 +314,26 @@ build_32_64 (function_builder &b, const char *signature,
 	     mode_suffix_index mode64, bool force_direct_overloads = false)
 {
   for (unsigned int pi = 0; group.preds[pi] != NUM_PREDS; ++pi)
-    if (group.types[0][0] == NUM_TYPE_SUFFIXES)
-      {
-	gcc_assert (mode32 != MODE_none && mode64 != MODE_none);
-	build_one (b, signature, group, mode32, 0, pi,
-		   force_direct_overloads);
-	build_one (b, signature, group, mode64, 0, pi,
-		   force_direct_overloads);
-      }
-    else
-      for (unsigned int ti = 0; group.types[ti][0] != NUM_TYPE_SUFFIXES; ++ti)
+    for (unsigned int gi = 0; group.groups[gi] != NUM_GROUP_SUFFIXES; ++gi)
+      if (group.types[0][0] == NUM_TYPE_SUFFIXES)
 	{
-	  unsigned int bits = type_suffixes[group.types[ti][0]].element_bits;
-	  gcc_assert (bits == 32 || bits == 64);
-	  mode_suffix_index mode = bits == 32 ? mode32 : mode64;
-	  if (mode != MODE_none)
-	    build_one (b, signature, group, mode, ti, pi,
-		       force_direct_overloads);
+	  gcc_assert (mode32 != MODE_none && mode64 != MODE_none);
+	  build_one (b, signature, group, mode32, 0, gi, pi,
+		     force_direct_overloads);
+	  build_one (b, signature, group, mode64, 0, gi, pi,
+		     force_direct_overloads);
 	}
+      else
+	for (unsigned int ti = 0; group.types[ti][0] != NUM_TYPE_SUFFIXES;
+	     ++ti)
+	  {
+	    unsigned int bits = type_suffixes[group.types[ti][0]].element_bits;
+	    gcc_assert (bits == 32 || bits == 64);
+	    mode_suffix_index mode = bits == 32 ? mode32 : mode64;
+	    if (mode != MODE_none)
+	      build_one (b, signature, group, mode, ti, gi, pi,
+			 force_direct_overloads);
+	  }
 }
 
 /* For every type and predicate combination in GROUP, add one function
@@ -423,10 +427,11 @@ build_all (function_builder &b, const char *signature,
 	   bool force_direct_overloads = false)
 {
   for (unsigned int pi = 0; group.preds[pi] != NUM_PREDS; ++pi)
-    for (unsigned int ti = 0;
-	 ti == 0 || group.types[ti][0] != NUM_TYPE_SUFFIXES; ++ti)
-      build_one (b, signature, group, mode_suffix_id, ti, pi,
-		 force_direct_overloads);
+    for (unsigned int gi = 0; group.groups[gi] != NUM_GROUP_SUFFIXES; ++gi)
+      for (unsigned int ti = 0;
+	   ti == 0 || group.types[ti][0] != NUM_TYPE_SUFFIXES; ++ti)
+	build_one (b, signature, group, mode_suffix_id, ti, gi, pi,
+		   force_direct_overloads);
 }
 
 /* TYPE is the largest type suffix associated with the arguments of R,
diff --git a/gcc/config/aarch64/aarch64-sve-builtins-sve2.cc b/gcc/config/aarch64/aarch64-sve-builtins-sve2.cc
index e066f096d..a94e5e269 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins-sve2.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins-sve2.cc
@@ -252,7 +252,7 @@ public:
 	       that we can use for sensible shift amounts.  */
 	    function_instance instance ("svqshl", functions::svqshl,
 					shapes::binary_int_opt_n, MODE_n,
-					f.type_suffix_ids, f.pred);
+					f.type_suffix_ids, GROUP_none, f.pred);
 	    return f.redirect_call (instance);
 	  }
 	else
@@ -261,7 +261,7 @@ public:
 	       that we can use for sensible shift amounts.  */
 	    function_instance instance ("svrshl", functions::svrshl,
 					shapes::binary_int_opt_n, MODE_n,
-					f.type_suffix_ids, f.pred);
+					f.type_suffix_ids, GROUP_none, f.pred);
 	    return f.redirect_call (instance);
 	  }
       }
@@ -290,7 +290,7 @@ public:
 				       -wi::to_wide (amount));
 	    function_instance instance ("svasr", functions::svasr,
 					shapes::binary_uint_opt_n, MODE_n,
-					f.type_suffix_ids, f.pred);
+					f.type_suffix_ids, GROUP_none, f.pred);
 	    if (f.type_suffix (0).unsigned_p)
 	      {
 		instance.base_name = "svlsr";
@@ -322,7 +322,7 @@ public:
 	       that we can use for sensible shift amounts.  */
 	    function_instance instance ("svlsl", functions::svlsl,
 					shapes::binary_uint_opt_n, MODE_n,
-					f.type_suffix_ids, f.pred);
+					f.type_suffix_ids, GROUP_none, f.pred);
 	    gcall *call = as_a <gcall *> (f.redirect_call (instance));
 	    gimple_call_set_arg (call, 2, amount);
 	    return call;
@@ -335,7 +335,7 @@ public:
 				       -wi::to_wide (amount));
 	    function_instance instance ("svrshr", functions::svrshr,
 					shapes::shift_right_imm, MODE_n,
-					f.type_suffix_ids, f.pred);
+					f.type_suffix_ids, GROUP_none, f.pred);
 	    gcall *call = as_a <gcall *> (f.redirect_call (instance));
 	    gimple_call_set_arg (call, 2, amount);
 	    return call;
diff --git a/gcc/config/aarch64/aarch64-sve-builtins.cc b/gcc/config/aarch64/aarch64-sve-builtins.cc
index dde01f676..dc3fd80da 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins.cc
+++ b/gcc/config/aarch64/aarch64-sve-builtins.cc
@@ -144,6 +144,13 @@ CONSTEXPR const type_suffix_info type_suffixes[NUM_TYPE_SUFFIXES + 1] = {
     0, VOIDmode }
 };
 
+CONSTEXPR const group_suffix_info group_suffixes[] = {
+#define DEF_SVE_GROUP_SUFFIX(NAME, VG, VECTORS_PER_TUPLE) \
+  { "_" #NAME, VG, VECTORS_PER_TUPLE },
+#include "aarch64-sve-builtins.def"
+  { "", 0, 1 }
+};
+
 /* Define a TYPES_<combination> macro for each combination of type
    suffixes that an ACLE function can have, where <combination> is the
    name used in DEF_SVE_FUNCTION entries.
@@ -483,6 +490,10 @@ DEF_SVE_TYPES_ARRAY (inc_dec_n);
 DEF_SVE_TYPES_ARRAY (reinterpret);
 DEF_SVE_TYPES_ARRAY (while);
 
+static const group_suffix_index groups_none[] = {
+  GROUP_none, NUM_GROUP_SUFFIXES
+};
+
 /* Used by functions that have no governing predicate.  */
 static const predication_index preds_none[] = { PRED_none, NUM_PREDS };
 
@@ -524,8 +535,8 @@ static const predication_index preds_z[] = { PRED_z, NUM_PREDS };
 /* A list of all SVE ACLE functions.  */
 static CONSTEXPR const function_group_info function_groups[] = {
 #define DEF_SVE_FUNCTION(NAME, SHAPE, TYPES, PREDS) \
-  { #NAME, &functions::NAME, &shapes::SHAPE, types_##TYPES, preds_##PREDS, \
-    REQUIRED_EXTENSIONS },
+  { #NAME, &functions::NAME, &shapes::SHAPE, types_##TYPES, groups_none, \
+    preds_##PREDS, REQUIRED_EXTENSIONS },
 #include "aarch64-sve-builtins.def"
 };
 
@@ -788,6 +799,7 @@ function_instance::hash () const
   h.add_int (mode_suffix_id);
   h.add_int (type_suffix_ids[0]);
   h.add_int (type_suffix_ids[1]);
+  h.add_int (group_suffix_id);
   h.add_int (pred);
   return h.end ();
 }
@@ -957,6 +969,8 @@ function_builder::get_name (const function_instance &instance,
   for (unsigned int i = 0; i < 2; ++i)
     if (!overloaded_p || instance.shape->explicit_type_suffix_p (i))
       append_name (instance.type_suffix (i).string);
+  if (!overloaded_p || instance.shape->explicit_group_suffix_p ())
+    append_name (instance.group_suffix ().string);
   append_name (pred_suffixes[instance.pred]);
   return finish_name ();
 }
@@ -1113,19 +1127,26 @@ void
 function_builder::add_overloaded_functions (const function_group_info &group,
 					    mode_suffix_index mode)
 {
-  unsigned int explicit_type0 = (*group.shape)->explicit_type_suffix_p (0);
-  unsigned int explicit_type1 = (*group.shape)->explicit_type_suffix_p (1);
-  for (unsigned int pi = 0; group.preds[pi] != NUM_PREDS; ++pi)
+  bool explicit_type0 = (*group.shape)->explicit_type_suffix_p (0);
+  bool explicit_type1 = (*group.shape)->explicit_type_suffix_p (1);
+  bool explicit_group = (*group.shape)->explicit_group_suffix_p ();
+  auto add_function = [&](const type_suffix_pair &types,
+			  group_suffix_index group_suffix_id,
+			  unsigned int pi)
+    {
+      function_instance instance (group.base_name, *group.base,
+				  *group.shape, mode, types,
+				  group_suffix_id, group.preds[pi]);
+      add_overloaded_function (instance, group.required_extensions);
+    };
+
+  auto add_group_suffix = [&](group_suffix_index group_suffix_id,
+			      unsigned int pi)
     {
       if (!explicit_type0 && !explicit_type1)
-	{
-	  /* Deal with the common case in which there is one overloaded
-	     function for all type combinations.  */
-	  function_instance instance (group.base_name, *group.base,
-				      *group.shape, mode, types_none[0],
-				      group.preds[pi]);
-	  add_overloaded_function (instance, group.required_extensions);
-	}
+	/* Deal with the common case in which there is one overloaded
+	   function for all type combinations.  */
+	add_function (types_none[0], group_suffix_id, pi);
       else
 	for (unsigned int ti = 0; group.types[ti][0] != NUM_TYPE_SUFFIXES;
 	     ++ti)
@@ -1136,12 +1157,16 @@ function_builder::add_overloaded_functions (const function_group_info &group,
 	      explicit_type0 ? group.types[ti][0] : NUM_TYPE_SUFFIXES,
 	      explicit_type1 ? group.types[ti][1] : NUM_TYPE_SUFFIXES
 	    };
-	    function_instance instance (group.base_name, *group.base,
-					*group.shape, mode, types,
-					group.preds[pi]);
-	    add_overloaded_function (instance, group.required_extensions);
+	    add_function (types, group_suffix_id, pi);
 	  }
-    }
+    };
+
+  for (unsigned int pi = 0; group.preds[pi] != NUM_PREDS; ++pi)
+    if (explicit_group)
+      for (unsigned int gi = 0; group.groups[gi] != NUM_GROUP_SUFFIXES; ++gi)
+	add_group_suffix (group.groups[gi], pi);
+    else
+      add_group_suffix (GROUP_none, pi);
 }
 
 /* Register all the functions in GROUP.  */
@@ -1213,29 +1238,34 @@ function_resolver::report_no_such_form (type_suffix_index type)
 }
 
 /* Silently check whether there is an instance of the function with the
-   mode suffix given by MODE and the type suffixes given by TYPE0 and TYPE1.
-   Return its function decl if so, otherwise return null.  */
+   mode suffix given by MODE, the type suffixes given by TYPE0 and TYPE1,
+   and the group suffix given by GROUP.  Return its function decl if so,
+   otherwise return null.  */
 tree
 function_resolver::lookup_form (mode_suffix_index mode,
 				type_suffix_index type0,
-				type_suffix_index type1)
+				type_suffix_index type1,
+				group_suffix_index group)
 {
   type_suffix_pair types = { type0, type1 };
-  function_instance instance (base_name, base, shape, mode, types, pred);
+  function_instance instance (base_name, base, shape, mode, types,
+			      group, pred);
   registered_function *rfn
     = function_table->find_with_hash (instance, instance.hash ());
   return rfn ? rfn->decl : NULL_TREE;
 }
 
-/* Resolve the function to one with the mode suffix given by MODE and the
-   type suffixes given by TYPE0 and TYPE1.  Return its function decl on
-   success, otherwise report an error and return error_mark_node.  */
+/* Resolve the function to one with the mode suffix given by MODE, the
+   type suffixes given by TYPE0 and TYPE1, and group suffix given by
+   GROUP.  Return its function decl on success, otherwise report an
+   error and return error_mark_node.  */
 tree
 function_resolver::resolve_to (mode_suffix_index mode,
 			       type_suffix_index type0,
-			       type_suffix_index type1)
+			       type_suffix_index type1,
+			       group_suffix_index group)
 {
-  tree res = lookup_form (mode, type0, type1);
+  tree res = lookup_form (mode, type0, type1, group);
   if (!res)
     {
       if (type1 == NUM_TYPE_SUFFIXES)
diff --git a/gcc/config/aarch64/aarch64-sve-builtins.def b/gcc/config/aarch64/aarch64-sve-builtins.def
index 6e4dcdbc9..d9bf9c350 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins.def
+++ b/gcc/config/aarch64/aarch64-sve-builtins.def
@@ -29,6 +29,10 @@
 #define DEF_SVE_TYPE_SUFFIX(A, B, C, D, E)
 #endif
 
+#ifndef DEF_SVE_GROUP_SUFFIX
+#define DEF_SVE_GROUP_SUFFIX(A, B, C)
+#endif
+
 #ifndef DEF_SVE_FUNCTION
 #define DEF_SVE_FUNCTION(A, B, C, D)
 #endif
@@ -95,10 +99,15 @@ DEF_SVE_TYPE_SUFFIX (u16, svuint16_t, unsigned, 16, VNx8HImode)
 DEF_SVE_TYPE_SUFFIX (u32, svuint32_t, unsigned, 32, VNx4SImode)
 DEF_SVE_TYPE_SUFFIX (u64, svuint64_t, unsigned, 64, VNx2DImode)
 
+DEF_SVE_GROUP_SUFFIX (x2, 0, 2)
+DEF_SVE_GROUP_SUFFIX (x3, 0, 3)
+DEF_SVE_GROUP_SUFFIX (x4, 0, 4)
+
 #include "aarch64-sve-builtins-base.def"
 #include "aarch64-sve-builtins-sve2.def"
 
 #undef DEF_SVE_FUNCTION
+#undef DEF_SVE_GROUP_SUFFIX
 #undef DEF_SVE_TYPE_SUFFIX
 #undef DEF_SVE_TYPE
 #undef DEF_SVE_MODE
diff --git a/gcc/config/aarch64/aarch64-sve-builtins.h b/gcc/config/aarch64/aarch64-sve-builtins.h
index 824c31cd7..374c57e93 100644
--- a/gcc/config/aarch64/aarch64-sve-builtins.h
+++ b/gcc/config/aarch64/aarch64-sve-builtins.h
@@ -180,6 +180,17 @@ enum type_suffix_index
   NUM_TYPE_SUFFIXES
 };
 
+/* Enumerates the possible group suffixes.  Each suffix combines two
+   optional pieces of information: the vector group size in a ZA index,
+   and the number of vectors in the largest tuple argument.  */
+enum group_suffix_index
+{
+#define DEF_SVE_GROUP_SUFFIX(NAME, VG, VECTORS_PER_TUPLE) GROUP_##NAME,
+#include "aarch64-sve-builtins.def"
+  GROUP_none,
+  NUM_GROUP_SUFFIXES
+};
+
 /* Combines two type suffixes.  */
 typedef enum type_suffix_index type_suffix_pair[2];
 
@@ -237,6 +248,21 @@ struct type_suffix_info
   machine_mode vector_mode : 16;
 };
 
+/* Static information about a group suffix.  */
+struct group_suffix_info
+{
+  /* The suffix string itself.  */
+  const char *string;
+
+  /* If the suffix describes a vector group in a ZA index, this is the
+     size of that group, otherwise it is zero.  */
+  unsigned int vg;
+
+  /* The number of vectors in the largest (or only) tuple argument,
+     or 1 if the suffix does not convey this information.  */
+  unsigned int vectors_per_tuple;
+};
+
 /* Static information about a set of functions.  */
 struct function_group_info
 {
@@ -251,14 +277,16 @@ struct function_group_info
      shapes.  */
   const function_shape *const *shape;
 
-  /* A list of the available type suffixes, and of the available predication
-     types.  The function supports every combination of the two.
+  /* A list of the available type suffixes, group suffixes, and predication
+     types.  The function supports every combination of the three.
+
+     The list of type suffixes is terminated by two NUM_TYPE_SUFFIXES.
+     It is lexicographically ordered based on the index value.
 
-     The list of type suffixes is terminated by two NUM_TYPE_SUFFIXES
-     while the list of predication types is terminated by NUM_PREDS.
-     The list of type suffixes is lexicographically ordered based
-     on the index value.  */
+     The list of group suffixes is terminated by NUM_GROUP_SUFFIXES
+     and the list of predication types is terminated by NUM_PREDS.  */
   const type_suffix_pair *types;
+  const group_suffix_index *groups;
   const predication_index *preds;
 
   /* The architecture extensions that the functions require, as a set of
@@ -273,7 +301,8 @@ class GTY((user)) function_instance
 public:
   function_instance (const char *, const function_base *,
 		     const function_shape *, mode_suffix_index,
-		     const type_suffix_pair &, predication_index);
+		     const type_suffix_pair &, group_suffix_index,
+		     predication_index);
 
   bool operator== (const function_instance &) const;
   bool operator!= (const function_instance &) const;
@@ -294,6 +323,8 @@ public:
   units_index displacement_units () const;
 
   const type_suffix_info &type_suffix (unsigned int) const;
+  const group_suffix_info &group_suffix () const;
+
   tree scalar_type (unsigned int) const;
   tree vector_type (unsigned int) const;
   tree tuple_type (unsigned int) const;
@@ -301,14 +332,14 @@ public:
   machine_mode vector_mode (unsigned int) const;
   machine_mode gp_mode (unsigned int) const;
 
-  /* The properties of the function.  (The explicit "enum"s are required
-     for gengtype.)  */
+  /* The properties of the function.  */
   const char *base_name;
   const function_base *base;
   const function_shape *shape;
-  enum mode_suffix_index mode_suffix_id;
+  mode_suffix_index mode_suffix_id;
   type_suffix_pair type_suffix_ids;
-  enum predication_index pred;
+  group_suffix_index group_suffix_id;
+  predication_index pred;
 };
 
 class registered_function;
@@ -390,10 +421,12 @@ public:
   tree report_no_such_form (type_suffix_index);
   tree lookup_form (mode_suffix_index,
 		    type_suffix_index = NUM_TYPE_SUFFIXES,
-		    type_suffix_index = NUM_TYPE_SUFFIXES);
+		    type_suffix_index = NUM_TYPE_SUFFIXES,
+		    group_suffix_index = GROUP_none);
   tree resolve_to (mode_suffix_index,
 		   type_suffix_index = NUM_TYPE_SUFFIXES,
-		   type_suffix_index = NUM_TYPE_SUFFIXES);
+		   type_suffix_index = NUM_TYPE_SUFFIXES,
+		   group_suffix_index = GROUP_none);
 
   type_suffix_index infer_integer_scalar_type (unsigned int);
   type_suffix_index infer_pointer_type (unsigned int, bool = false);
@@ -641,6 +674,11 @@ class function_shape
 public:
   virtual bool explicit_type_suffix_p (unsigned int) const = 0;
 
+  /* True if the group suffix is present in overloaded names.
+     This isn't meaningful for pre-SME intrinsics, and true is
+     more common than false, so provide a default definition.  */
+  virtual bool explicit_group_suffix_p () const { return true; }
+
   /* Define all functions associated with the given group.  */
   virtual void build (function_builder &,
 		      const function_group_info &) const = 0;
@@ -669,6 +707,7 @@ private:
 
 extern const type_suffix_info type_suffixes[NUM_TYPE_SUFFIXES + 1];
 extern const mode_suffix_info mode_suffixes[MODE_none + 1];
+extern const group_suffix_info group_suffixes[NUM_GROUP_SUFFIXES];
 
 extern tree scalar_types[NUM_VECTOR_TYPES];
 extern tree acle_vector_types[MAX_TUPLE_SIZE][NUM_VECTOR_TYPES + 1];
@@ -728,9 +767,11 @@ function_instance (const char *base_name_in,
 		   const function_shape *shape_in,
 		   mode_suffix_index mode_suffix_id_in,
 		   const type_suffix_pair &type_suffix_ids_in,
+		   group_suffix_index group_suffix_id_in,
 		   predication_index pred_in)
   : base_name (base_name_in), base (base_in), shape (shape_in),
-    mode_suffix_id (mode_suffix_id_in), pred (pred_in)
+    mode_suffix_id (mode_suffix_id_in), group_suffix_id (group_suffix_id_in),
+    pred (pred_in)
 {
   memcpy (type_suffix_ids, type_suffix_ids_in, sizeof (type_suffix_ids));
 }
@@ -741,9 +782,10 @@ function_instance::operator== (const function_instance &other) const
   return (base == other.base
 	  && shape == other.shape
 	  && mode_suffix_id == other.mode_suffix_id
-	  && pred == other.pred
 	  && type_suffix_ids[0] == other.type_suffix_ids[0]
-	  && type_suffix_ids[1] == other.type_suffix_ids[1]);
+	  && type_suffix_ids[1] == other.type_suffix_ids[1]
+	  && group_suffix_id == other.group_suffix_id
+	  && pred == other.pred);
 }
 
 inline bool
@@ -815,6 +857,13 @@ function_instance::type_suffix (unsigned int i) const
   return type_suffixes[type_suffix_ids[i]];
 }
 
+/* Return information about the function's group suffix.  */
+inline const group_suffix_info &
+function_instance::group_suffix () const
+{
+  return group_suffixes[group_suffix_id];
+}
+
 /* Return the scalar type associated with type suffix I.  */
 inline tree
 function_instance::scalar_type (unsigned int i) const
-- 
2.33.0