[AArch64] Workaround for Cortex A53 erratum 843419

Some early revisions of the Cortex-A53 have an erratum (843419).  The
details of the erratum are quite complex and involve dynamic
conditions.  For the purposes of the workaround we have simplified the
static conditions to an ADRP in the last two instructions of a 4KByte
page, followed within four instructions by a load/store dependent on
the ADRP.

This patch adds support to conservatively scan for and workaround
Cortex A53 erratum 843419.  There are two different workaround
strategies used.  The first is to rewrite ADRP instructions which form
part of an erratum sequence with an ADR instruction.  In situations
where the ADR provides insufficient offset the dependent load or store
instruction from the sequence is moved to a stub section and branches
are inserted from the original sequence to the relocated instruction
and back again.

Stub section sizes are rounded up to a multiple of 4096 in order to
ensure that the act of inserting work around stubs does not create
more errata sequences.

Workaround stubs are always inserted into the stub section associated
with the input section containing the erratum sequence.  This ensures
that the fully relocated form of the veneered load store instruction
is available at the point in time when the stub section is written.

1 files changed, 433 insertions, 5 deletions

diff --git a/bfd/elfnn-aarch64.c b/bfd/elfnn-aarch64.c
index e0e4915..85cf856 100644
--- a/bfd/elfnn-aarch64.c
+++ b/bfd/elfnn-aarch64.c
@@ -1640,6 +1640,12 @@ static const uint32_t aarch64_erratum_835769_stub[] =
   0x14000000,    /* b <label> */
 };
 
+static const uint32_t aarch64_erratum_843419_stub[] =
+{
+  0x00000000,    /* Placeholder for LDR instruction.  */
+  0x14000000,    /* b <label> */
+};
+
 /* Section name for stubs is the associated section name plus this
    string.  */
 #define STUB_SUFFIX ".stub"
@@ -1650,6 +1656,7 @@ enum elf_aarch64_stub_type
   aarch64_stub_adrp_branch,
   aarch64_stub_long_branch,
   aarch64_stub_erratum_835769_veneer,
+  aarch64_stub_erratum_843419_veneer,
 };
 
 struct elf_aarch64_stub_hash_entry
@@ -1688,6 +1695,9 @@ struct elf_aarch64_stub_hash_entry
   /* The instruction which caused this stub to be generated (only valid for
      erratum 835769 workaround stubs at present).  */
   uint32_t veneered_insn;
+
+  /* In an erratum 843419 workaround stub, the ADRP instruction offset.  */
+  bfd_vma adrp_offset;
 };
 
 /* Used to build a map of a section.  This is required for mixed-endian
@@ -1836,6 +1846,12 @@ struct elf_aarch64_link_hash_table
   /* Fix erratum 835769.  */
   int fix_erratum_835769;
 
+  /* Fix erratum 843419.  */
+  int fix_erratum_843419;
+
+  /* Enable ADRP->ADR rewrite for erratum 843419 workaround.  */
+  int fix_erratum_843419_adr;
+
   /* The number of bytes in the initial entry in the PLT.  */
   bfd_size_type plt_header_size;
 
@@ -1957,6 +1973,7 @@ stub_hash_newfunc (struct bfd_hash_entry *entry,
 
       /* Initialize the local fields.  */
       eh = (struct elf_aarch64_stub_hash_entry *) entry;
+      eh->adrp_offset = 0;
       eh->stub_sec = NULL;
       eh->stub_offset = 0;
       eh->target_value = 0;
@@ -2399,6 +2416,34 @@ _bfd_aarch64_add_stub_entry_in_group (const char *stub_name,
   return stub_entry;
 }
 
+/* Add a new stub entry in the final stub section to the stub hash.
+   Not all fields of the new stub entry are initialised.  */
+
+static struct elf_aarch64_stub_hash_entry *
+_bfd_aarch64_add_stub_entry_after (const char *stub_name,
+				   asection *link_section,
+				   struct elf_aarch64_link_hash_table *htab)
+{
+  asection *stub_sec;
+  struct elf_aarch64_stub_hash_entry *stub_entry;
+
+  stub_sec = _bfd_aarch64_get_stub_for_link_section (link_section, htab);
+  stub_entry = aarch64_stub_hash_lookup (&htab->stub_hash_table, stub_name,
+					 TRUE, FALSE);
+  if (stub_entry == NULL)
+    {
+      (*_bfd_error_handler) (_("cannot create stub entry %s"), stub_name);
+      return NULL;
+    }
+
+  stub_entry->stub_sec = stub_sec;
+  stub_entry->stub_offset = 0;
+  stub_entry->id_sec = link_section;
+
+  return stub_entry;
+}
+
+
 static bfd_boolean
 aarch64_build_one_stub (struct bfd_hash_entry *gen_entry,
 			void *in_arg ATTRIBUTE_UNUSED)
@@ -2455,6 +2500,10 @@ aarch64_build_one_stub (struct bfd_hash_entry *gen_entry,
       template = aarch64_erratum_835769_stub;
       template_size = sizeof (aarch64_erratum_835769_stub);
       break;
+    case aarch64_stub_erratum_843419_veneer:
+      template = aarch64_erratum_843419_stub;
+      template_size = sizeof (aarch64_erratum_843419_stub);
+      break;
     default:
       abort ();
     }
@@ -2506,6 +2555,12 @@ aarch64_build_one_stub (struct bfd_hash_entry *gen_entry,
 		  stub_sec->contents + stub_entry->stub_offset + 4);
       break;
 
+    case aarch64_stub_erratum_843419_veneer:
+      if (aarch64_relocate (AARCH64_R (JUMP26), stub_bfd, stub_sec,
+			    stub_entry->stub_offset + 4, sym_value + 4))
+	BFD_FAIL ();
+      break;
+
     default:
       abort ();
     }
@@ -2537,6 +2592,9 @@ aarch64_size_one_stub (struct bfd_hash_entry *gen_entry,
     case aarch64_stub_erratum_835769_veneer:
       size = sizeof (aarch64_erratum_835769_stub);
       break;
+    case aarch64_stub_erratum_843419_veneer:
+      size = sizeof (aarch64_erratum_843419_stub);
+      break;
     default:
       abort ();
     }
@@ -3001,7 +3059,7 @@ _bfd_aarch64_erratum_835769_stub_name (unsigned num_fixes)
   return stub_name;
 }
 
-/* Scan for cortex-a53 erratum 835769 sequence.
+/* Scan for Cortex-A53 erratum 835769 sequence.
 
    Return TRUE else FALSE on abnormal termination.  */
 
@@ -3091,6 +3149,82 @@ _bfd_aarch64_erratum_835769_scan (bfd *input_bfd,
 }
 
 
+/* Test if instruction INSN is ADRP.  */
+
+static bfd_boolean
+_bfd_aarch64_adrp_p (uint32_t insn)
+{
+  return ((insn & 0x9f000000) == 0x90000000);
+}
+
+
+/* Helper predicate to look for cortex-a53 erratum 843419 sequence 1.  */
+
+static bfd_boolean
+_bfd_aarch64_erratum_843419_sequence_p (uint32_t insn_1, uint32_t insn_2,
+					uint32_t insn_3)
+{
+  uint32_t rt;
+  uint32_t rt2;
+  bfd_boolean pair;
+  bfd_boolean load;
+
+  return (aarch64_mem_op_p (insn_2, &rt, &rt2, &pair, &load)
+	  && (!pair
+	      || (pair && !load))
+	  && AARCH64_LDST_UIMM (insn_3)
+	  && AARCH64_RN (insn_3) == AARCH64_RD (insn_1));
+}
+
+
+/* Test for the presence of Cortex-A53 erratum 843419 instruction sequence.
+
+   Return TRUE if section CONTENTS at offset I contains one of the
+   erratum 843419 sequences, otherwise return FALSE.  If a sequence is
+   seen set P_VENEER_I to the offset of the final LOAD/STORE
+   instruction in the sequence.
+ */
+
+static bfd_boolean
+_bfd_aarch64_erratum_843419_p (bfd_byte *contents, bfd_vma vma,
+			       bfd_vma i, bfd_vma span_end,
+			       bfd_vma *p_veneer_i)
+{
+  uint32_t insn_1 = bfd_getl32 (contents + i);
+
+  if (!_bfd_aarch64_adrp_p (insn_1))
+    return FALSE;
+
+  if (span_end < i + 12)
+    return FALSE;
+
+  uint32_t insn_2 = bfd_getl32 (contents + i + 4);
+  uint32_t insn_3 = bfd_getl32 (contents + i + 8);
+
+  if ((vma & 0xfff) != 0xff8 && (vma & 0xfff) != 0xffc)
+    return FALSE;
+
+  if (_bfd_aarch64_erratum_843419_sequence_p (insn_1, insn_2, insn_3))
+    {
+      *p_veneer_i = i + 8;
+      return TRUE;
+    }
+
+  if (span_end < i + 16)
+    return FALSE;
+
+  uint32_t insn_4 = bfd_getl32 (contents + i + 12);
+
+  if (_bfd_aarch64_erratum_843419_sequence_p (insn_1, insn_2, insn_4))
+    {
+      *p_veneer_i = i + 12;
+      return TRUE;
+    }
+
+  return FALSE;
+}
+
+
 /* Resize all stub sections.  */
 
 static void
@@ -3119,9 +3253,174 @@ _bfd_aarch64_resize_stubs (struct elf_aarch64_link_hash_table *htab)
 
       if (section->size)
 	section->size += 4;
+
+      /* Ensure all stub sections have a size which is a multiple of
+	 4096.  This is important in order to ensure that the insertion
+	 of stub sections does not in itself move existing code around
+	 in such a way that new errata sequences are created.  */
+      if (htab->fix_erratum_843419)
+	if (section->size)
+	  section->size = BFD_ALIGN (section->size, 0x1000);
     }
 }
 
+
+/* Construct an erratum 843419 workaround stub name.
+ */
+
+static char *
+_bfd_aarch64_erratum_843419_stub_name (asection *input_section,
+				       bfd_vma offset)
+{
+  const bfd_size_type len = 8 + 4 + 1 + 8 + 1 + 16 + 1;
+  char *stub_name = bfd_malloc (len);
+
+  if (stub_name != NULL)
+    snprintf (stub_name, len, "e843419@%04x_%08x_%" BFD_VMA_FMT "x",
+	      input_section->owner->id,
+	      input_section->id,
+	      offset);
+  return stub_name;
+}
+
+/*  Build a stub_entry structure describing an 843419 fixup.
+
+    The stub_entry constructed is populated with the bit pattern INSN
+    of the instruction located at OFFSET within input SECTION.
+
+    Returns TRUE on success.  */
+
+static bfd_boolean
+_bfd_aarch64_erratum_843419_fixup (uint32_t insn,
+				   bfd_vma adrp_offset,
+				   bfd_vma ldst_offset,
+				   asection *section,
+				   struct bfd_link_info *info)
+{
+  struct elf_aarch64_link_hash_table *htab = elf_aarch64_hash_table (info);
+  char *stub_name;
+  struct elf_aarch64_stub_hash_entry *stub_entry;
+
+  stub_name = _bfd_aarch64_erratum_843419_stub_name (section, ldst_offset);
+  stub_entry = aarch64_stub_hash_lookup (&htab->stub_hash_table, stub_name,
+					 FALSE, FALSE);
+  if (stub_entry)
+    {
+      free (stub_name);
+      return TRUE;
+    }
+
+  /* We always place an 843419 workaround veneer in the stub section
+     attached to the input section in which an erratum sequence has
+     been found.  This ensures that later in the link process (in
+     elfNN_aarch64_write_section) when we copy the veneered
+     instruction from the input section into the stub section the
+     copied instruction will have had any relocations applied to it.
+     If we placed workaround veneers in any other stub section then we
+     could not assume that all relocations have been processed on the
+     corresponding input section at the point we output the stub
+     section.
+   */
+
+  stub_entry = _bfd_aarch64_add_stub_entry_after (stub_name, section, htab);
+  if (stub_entry == NULL)
+    {
+      free (stub_name);
+      return FALSE;
+    }
+
+  stub_entry->adrp_offset = adrp_offset;
+  stub_entry->target_value = ldst_offset;
+  stub_entry->target_section = section;
+  stub_entry->stub_type = aarch64_stub_erratum_843419_veneer;
+  stub_entry->veneered_insn = insn;
+  stub_entry->output_name = stub_name;
+
+  return TRUE;
+}
+
+
+/* Scan an input section looking for the signature of erratum 843419.
+
+   Scans input SECTION in INPUT_BFD looking for erratum 843419
+   signatures, for each signature found a stub_entry is created
+   describing the location of the erratum for subsequent fixup.
+
+   Return TRUE on successful scan, FALSE on failure to scan.
+ */
+
+static bfd_boolean
+_bfd_aarch64_erratum_843419_scan (bfd *input_bfd, asection *section,
+				  struct bfd_link_info *info)
+{
+  struct elf_aarch64_link_hash_table *htab = elf_aarch64_hash_table (info);
+
+  if (htab == NULL)
+    return TRUE;
+
+  if (elf_section_type (section) != SHT_PROGBITS
+      || (elf_section_flags (section) & SHF_EXECINSTR) == 0
+      || (section->flags & SEC_EXCLUDE) != 0
+      || (section->sec_info_type == SEC_INFO_TYPE_JUST_SYMS)
+      || (section->output_section == bfd_abs_section_ptr))
+    return TRUE;
+
+  do
+    {
+      bfd_byte *contents = NULL;
+      struct _aarch64_elf_section_data *sec_data;
+      unsigned int span;
+
+      if (elf_section_data (section)->this_hdr.contents != NULL)
+	contents = elf_section_data (section)->this_hdr.contents;
+      else if (! bfd_malloc_and_get_section (input_bfd, section, &contents))
+	return FALSE;
+
+      sec_data = elf_aarch64_section_data (section);
+
+      qsort (sec_data->map, sec_data->mapcount,
+	     sizeof (elf_aarch64_section_map), elf_aarch64_compare_mapping);
+
+      for (span = 0; span < sec_data->mapcount; span++)
+	{
+	  unsigned int span_start = sec_data->map[span].vma;
+	  unsigned int span_end = ((span == sec_data->mapcount - 1)
+				   ? sec_data->map[0].vma + section->size
+				   : sec_data->map[span + 1].vma);
+	  unsigned int i;
+	  char span_type = sec_data->map[span].type;
+
+	  if (span_type == 'd')
+	    continue;
+
+	  for (i = span_start; i + 8 < span_end; i += 4)
+	    {
+	      bfd_vma vma = (section->output_section->vma
+			     + section->output_offset
+			     + i);
+	      bfd_vma veneer_i;
+
+	      if (_bfd_aarch64_erratum_843419_p
+		  (contents, vma, i, span_end, &veneer_i))
+		{
+		  uint32_t insn = bfd_getl32 (contents + veneer_i);
+
+		  if (!_bfd_aarch64_erratum_843419_fixup (insn, i, veneer_i,
+							  section, info))
+		    return FALSE;
+		}
+	    }
+	}
+
+      if (elf_section_data (section)->this_hdr.contents == NULL)
+	free (contents);
+    }
+  while (0);
+
+  return TRUE;
+}
+
+
 /* Determine and set the size of the stub section for a final link.
 
    The basic idea here is to examine all the relocations looking for
@@ -3167,6 +3466,8 @@ elfNN_aarch64_size_stubs (bfd *output_bfd,
 
   group_sections (htab, stub_group_size, stubs_always_before_branch);
 
+  (*htab->layout_sections_again) ();
+
   if (htab->fix_erratum_835769)
     {
       bfd *input_bfd;
@@ -3177,7 +3478,29 @@ elfNN_aarch64_size_stubs (bfd *output_bfd,
 					       &num_erratum_835769_fixes))
 	  return FALSE;
 
-      stub_changed = TRUE;
+      _bfd_aarch64_resize_stubs (htab);
+      (*htab->layout_sections_again) ();
+    }
+
+  if (htab->fix_erratum_843419)
+    {
+      bfd *input_bfd;
+
+      for (input_bfd = info->input_bfds;
+	   input_bfd != NULL;
+	   input_bfd = input_bfd->link.next)
+	{
+	  asection *section;
+
+	  for (section = input_bfd->sections;
+	       section != NULL;
+	       section = section->next)
+	    if (!_bfd_aarch64_erratum_843419_scan (input_bfd, section, info))
+	      return FALSE;
+	}
+
+      _bfd_aarch64_resize_stubs (htab);
+      (*htab->layout_sections_again) ();
     }
 
   while (1)
@@ -3582,13 +3905,16 @@ bfd_elfNN_aarch64_set_options (struct bfd *output_bfd,
 			       struct bfd_link_info *link_info,
 			       int no_enum_warn,
 			       int no_wchar_warn, int pic_veneer,
-			       int fix_erratum_835769)
+			       int fix_erratum_835769,
+			       int fix_erratum_843419)
 {
   struct elf_aarch64_link_hash_table *globals;
 
   globals = elf_aarch64_hash_table (link_info);
   globals->pic_veneer = pic_veneer;
   globals->fix_erratum_835769 = fix_erratum_835769;
+  globals->fix_erratum_843419 = fix_erratum_843419;
+  globals->fix_erratum_843419_adr = TRUE;
 
   BFD_ASSERT (is_aarch64_elf (output_bfd));
   elf_aarch64_tdata (output_bfd)->no_enum_size_warning = no_enum_warn;
@@ -3924,6 +4250,7 @@ symbol_tlsdesc_got_offset (bfd *input_bfd, struct elf_link_hash_entry *h,
 
 struct erratum_835769_branch_to_stub_data
 {
+  struct bfd_link_info *info;
   asection *output_section;
   bfd_byte *contents;
 };
@@ -3976,6 +4303,87 @@ make_branch_to_erratum_835769_stub (struct bfd_hash_entry *gen_entry,
   return TRUE;
 }
 
+
+static bfd_boolean
+_bfd_aarch64_erratum_843419_branch_to_stub (struct bfd_hash_entry *gen_entry,
+					    void *in_arg)
+{
+  struct elf_aarch64_stub_hash_entry *stub_entry
+    = (struct elf_aarch64_stub_hash_entry *) gen_entry;
+  struct erratum_835769_branch_to_stub_data *data
+    = (struct erratum_835769_branch_to_stub_data *) in_arg;
+  struct bfd_link_info *info;
+  struct elf_aarch64_link_hash_table *htab;
+  bfd_byte *contents;
+  asection *section;
+  bfd *abfd;
+  bfd_vma place;
+  uint32_t insn;
+
+  info = data->info;
+  contents = data->contents;
+  section = data->output_section;
+
+  htab = elf_aarch64_hash_table (info);
+
+  if (stub_entry->target_section != section
+      || stub_entry->stub_type != aarch64_stub_erratum_843419_veneer)
+    return TRUE;
+
+  insn = bfd_getl32 (contents + stub_entry->target_value);
+  bfd_putl32 (insn,
+	      stub_entry->stub_sec->contents + stub_entry->stub_offset);
+
+  place = (section->output_section->vma + section->output_offset
+	   + stub_entry->adrp_offset);
+  insn = bfd_getl32 (contents + stub_entry->adrp_offset);
+
+  if ((insn & AARCH64_ADRP_OP_MASK) !=  AARCH64_ADRP_OP)
+    abort ();
+
+  bfd_signed_vma imm =
+    (_bfd_aarch64_sign_extend
+     ((bfd_vma) _bfd_aarch64_decode_adrp_imm (insn) << 12, 33)
+     - (place & 0xfff));
+
+  if (htab->fix_erratum_843419_adr
+      && (imm >= AARCH64_MIN_ADRP_IMM  && imm <= AARCH64_MAX_ADRP_IMM))
+    {
+      insn = (_bfd_aarch64_reencode_adr_imm (AARCH64_ADR_OP, imm)
+	      | AARCH64_RT (insn));
+      bfd_putl32 (insn, contents + stub_entry->adrp_offset);
+    }
+  else
+    {
+      bfd_vma veneered_insn_loc;
+      bfd_vma veneer_entry_loc;
+      bfd_signed_vma branch_offset;
+      uint32_t branch_insn;
+
+      veneered_insn_loc = stub_entry->target_section->output_section->vma
+	+ stub_entry->target_section->output_offset
+	+ stub_entry->target_value;
+      veneer_entry_loc = stub_entry->stub_sec->output_section->vma
+	+ stub_entry->stub_sec->output_offset
+	+ stub_entry->stub_offset;
+      branch_offset = veneer_entry_loc - veneered_insn_loc;
+
+      abfd = stub_entry->target_section->owner;
+      if (!aarch64_valid_branch_p (veneer_entry_loc, veneered_insn_loc))
+	(*_bfd_error_handler)
+	  (_("%B: error: Erratum 843419 stub out "
+	     "of range (input file too large)"), abfd);
+
+      branch_insn = 0x14000000;
+      branch_offset >>= 2;
+      branch_offset &= 0x3ffffff;
+      branch_insn |= branch_offset;
+      bfd_putl32 (branch_insn, contents + stub_entry->target_value);
+    }
+  return TRUE;
+}
+
+
 static bfd_boolean
 elfNN_aarch64_write_section (bfd *output_bfd  ATTRIBUTE_UNUSED,
 			     struct bfd_link_info *link_info,
@@ -3994,12 +4402,24 @@ elfNN_aarch64_write_section (bfd *output_bfd  ATTRIBUTE_UNUSED,
     {
       struct erratum_835769_branch_to_stub_data data;
 
+      data.info = link_info;
       data.output_section = sec;
       data.contents = contents;
       bfd_hash_traverse (&globals->stub_hash_table,
 			 make_branch_to_erratum_835769_stub, &data);
     }
 
+  if (globals->fix_erratum_843419)
+    {
+      struct erratum_835769_branch_to_stub_data data;
+
+      data.info = link_info;
+      data.output_section = sec;
+      data.contents = contents;
+      bfd_hash_traverse (&globals->stub_hash_table,
+			 _bfd_aarch64_erratum_843419_branch_to_stub, &data);
+    }
+
   return FALSE;
 }
 
@@ -6461,6 +6881,14 @@ aarch64_map_one_stub (struct bfd_hash_entry *gen_entry, void *in_arg)
       if (!elfNN_aarch64_output_map_sym (osi, AARCH64_MAP_INSN, addr))
 	return FALSE;
       break;
+    case aarch64_stub_erratum_843419_veneer:
+      if (!elfNN_aarch64_output_stub_sym (osi, stub_name, addr,
+					  sizeof (aarch64_erratum_843419_stub)))
+	return FALSE;
+      if (!elfNN_aarch64_output_map_sym (osi, AARCH64_MAP_INSN, addr))
+	return FALSE;
+      break;
+
     default:
       abort ();
     }
@@ -7161,8 +7589,8 @@ elfNN_aarch64_size_dynamic_sections (bfd *output_bfd ATTRIBUTE_UNUSED,
     }
 
   /* Init mapping symbols information to use later to distingush between
-     code and data while scanning for erratam 835769.  */
-  if (htab->fix_erratum_835769)
+     code and data while scanning for errata.  */
+  if (htab->fix_erratum_835769 || htab->fix_erratum_843419)
     for (ibfd = info->input_bfds; ibfd != NULL; ibfd = ibfd->link.next)
       {
 	if (!is_aarch64_elf (ibfd))