cpu: add V3 BPF atomic instructions

This patch adds a set of new atomic instructions that were introduced
in the version 3 BPF ISA:

Atomic operations:

  xor{w,dw}
  xand{w,dw}
  xxor{w,dw}

Atomic fetch-and-operate:

  xfadd{w,dw}
  xfor{w,dw}
  xfand{w,dw}
  xfxor{w,dw}

Other:

  xchg{w,dw}
  xcmp{w,dw}

7 files changed, 593 insertions, 40 deletions

diff --git a/cpu/ChangeLog b/cpu/ChangeLog
index 23c89c9..78ed5bd 100644
--- a/cpu/ChangeLog
+++ b/cpu/ChangeLog
@@ -1,3 +1,7 @@
+2023-05-10  Jose E. Marchesi  <jose.marchesi@oracle.com>
+
+	* bpf.cpu: Add BPF V3 atomic instructions, along with semantics.
+
 2023-03-15  Nick Clifton  <nickc@redhat.com>
 
 	PR 30231
diff --git a/cpu/bpf.cpu b/cpu/bpf.cpu
index 2ae74fc..c252155 100644
--- a/cpu/bpf.cpu
+++ b/cpu/bpf.cpu
@@ -51,7 +51,7 @@
 ;; Whereas the 128-bit instructions (at the moment there is only one
 ;; of such instructions, lddw) have the form:
 ;;
-;;      code:8 regs:8 offset:16 imm:32 unused:32 imm:32 
+;;      code:8 regs:8 offset:16 imm:32 unused:32 imm:32
 ;;
 ;; In both formats `regs' is itself composed by two fields:
 ;;
@@ -83,7 +83,7 @@
 ;;
 ;; and another for big-endian with instructions like:
 ;;
-;;   code:8 dst:4 src:4 offset:16 imm:32 [unused:32 imm:32]
+;;   code:8 dst:4 src:4x offset:16 imm:32 [unused:32 imm:32]
 ;;
 ;; where `offset' and the immediate fields are encoded in
 ;; little-endian and big-endian byte-order, respectively.
@@ -206,7 +206,19 @@
   (type pc UDI)
   (get () (raw-reg h-pc))
   (set (newval) (set (raw-reg h-pc) newval)))
-  
+
+;; r0 register is used by compare and exchange atomic operation
+;; used in addressing by offset mode, so they compare/xchange
+;; values in memory with r0.
+
+(define-hardware
+  (name h-r0)
+  (comment "r0 register")
+  (attrs all-isas)
+  (type register DI)
+  (get () (raw-reg h-r0))
+  (set (newval) (set (raw-reg h-r0) newval)))
+
 ;; A 64-bit h-sint to be used by the imm64 operand below.  XXX this
 ;; shouldn't be needed, as h-sint is supposed to be able to hold
 ;; 64-bit values.  However, in practice CGEN limits h-sint to 32 bits
@@ -234,6 +246,7 @@
     (length x-length)
     (mode x-mode)))
 
+
 ;; For arithmetic and jump instructions the 8-bit code field is
 ;; subdivided in:
 ;;
@@ -274,11 +287,54 @@
 (dwf f-op-mode "eBPF opcode mode" (all-isas) 0 8 7 3 UINT)
 (dwf f-op-size "eBPF opcode size" (all-isas) 0 8 4 2 UINT)
 
+;; Atomic instructions abuse f-imm32 to hold extra opcodes.
+;; These opcodes are structured like:
+;;
+;;
+;;          31       imm32        0
+;;         +-----------+-----+-----+
+;;         |           | op  |flags|
+;;         +-----------+-----+-----+
+;;                      --4-- --4--
+;;
+;; Where:
+;;
+;;   OP is one of ADD, AND, OR, XOR, CHG, CMP.
+;;   FLAGS is either 0 or FETCH.
+
+(dwf f-op-atomic "eBPF atomic insn opcode" (all-isas) 32 64 31 32 UINT)
+
+(define-normal-insn-enum insn-atomic-op-le "eBPF atomic insn opcode"
+    ((ISA ebpfle xbpfle)) OP_ATOMIC_LE_ f-op-atomic
+      ((ADD #x00)
+       (OR  #x40)
+       (AND #x50)
+       (XOR #xa0)
+       (FADD #x01)
+       (FOR  #x41)
+       (FAND #x51)
+       (FXOR #xa1)
+       (CHG #xe1)
+       (CMP #xf1)))
+
+(define-normal-insn-enum insn-atomic-op-be "eBPF atomic insn opcode"
+    ((ISA ebpfbe xbpfbe)) OP_ATOMIC_BE_ f-op-atomic
+      ((ADD #x00000000)
+       (OR  #x40000000)
+       (AND #x50000000)
+       (XOR #xa0000000)
+       (FADD #x01000000)
+       (FOR  #x41000000)
+       (FAND #x51000000)
+       (FXOR #xa1000000)
+       (CHG #xe1000000)
+       (CMP #xf1000000)))
+
 (define-normal-insn-enum insn-op-mode "eBPF load/store instruction modes"
   (all-isas) OP_MODE_ f-op-mode
   ((IMM #b000) (ABS #b001) (IND #b010) (MEM #b011)
    ;; #b100 and #b101 are used in classic BPF only, reserved in eBPF.
-   (XADD #b110)))
+   (ATOMIC #b110)))
 
 (define-normal-insn-enum insn-op-size "eBPF load/store instruction sizes"
   (all-isas) OP_SIZE_ f-op-size
@@ -541,7 +597,7 @@
 ;; variant for each ISA:
 ;;
 ;;  LDDWle for the little-endian ISA
-;;  LDDWbe for the big-endian ISA  
+;;  LDDWbe for the big-endian ISA
 
 (define-pmacro (define-lddw x-endian)
   (dni (.sym lddw x-endian)
@@ -614,7 +670,7 @@
        ()))
 
 (define-pmacro (define-ldind x-endian)
-  (begin    
+  (begin
     (dlind "w" W x-endian SI)
     (dlind "h" H x-endian HI)
     (dlind "b" B x-endian QI)
@@ -810,35 +866,91 @@
 
 ;;; Atomic instructions
 
-;; The atomic exchange-and-add instructions come in two flavors: one
+;; The atomic exchange-and-op instructions come in two flavors: one
 ;; for swapping 64-bit quantities and another for 32-bit quantities.
 
-(define-pmacro (sem-exchange-and-add x-endian x-mode)
+;; Semantic routines for regular atomic operations.
+(define-pmacro (sem-atomic-op x-semop x-endian x-mode)
+  (sequence VOID ((x-mode tmp))
+            (set x-mode tmp (mem x-mode (add DI (.sym dst x-endian) offset16)))
+            (set x-mode
+                 (mem x-mode (add DI (.sym dst x-endian) offset16))
+                 (x-semop x-mode tmp (.sym src x-endian)))))
+
+;; Semantic routines for atomic operations that involve exchange.
+(define-pmacro (sem-atomic-op-fetch x-semop x-endian x-mode)
   (sequence VOID ((x-mode tmp))
-            ;; XXX acquire lock in simulator...  as a hardware element?
             (set x-mode tmp (mem x-mode (add DI (.sym dst x-endian) offset16)))
             (set x-mode
                  (mem x-mode (add DI (.sym dst x-endian) offset16))
-                 (add x-mode tmp (.sym src x-endian)))))
+                 (x-semop x-mode tmp (.sym src x-endian)))
+            (set x-mode (.sym src x-endian) tmp)))
+
+;; Semantic routine for the atomic exchange.
+(define-pmacro (sem-exchange x-semop x-endian x-mode)
+  (sequence VOID ((x-mode tmp))
+            (set x-mode tmp (mem x-mode (add DI (.sym dst x-endian) offset16)))
+            (set x-mode (mem x-mode (add DI (.sym dst x-endian) offset16)) (.sym src x-endian))
+            (set x-mode (.sym src x-endian) tmp)))
+
+;; Semantic routine for compare-and-exchange.
+(define-pmacro (sem-cmp-exchange x-semop x-endian x-mode)
+  (sequence VOID ((x-mode tmp))
+            (set x-mode tmp (mem x-mode (add DI (.sym dst x-endian) offset16)))
+            (if (eq x-mode (reg h-r0) tmp)
+                 (set x-mode (mem x-mode (add DI (.sym dst x-endian) offset16)) (.sym src x-endian)))
+            (set x-mode (reg h-r0) (zext tmp))))
+
+;; Atomic operation without fetching.
+(define-pmacro (dai x-basename x-suffix x-size x-op-code x-endian x-mode semproc semop)
+  (begin
+     (dni (.str "x" x-basename x-suffix x-endian)
+          (.str "x" x-basename x-suffix x-endian)
+          (endian-isas x-endian)
+          (.str "x" x-basename x-suffix " [$dst" x-endian "+$offset16],$src" x-endian)
+          (+ (.sym src x-endian) (.sym dst x-endian)
+             offset16 OP_MODE_ATOMIC x-size OP_CLASS_STX x-op-code)
+          (semproc semop x-endian x-mode)
+          ())))
+
+(define-pmacro (dais x-basename x-op-code x-endian semproc semop)
+  (begin
+    (dai x-basename "dw" OP_SIZE_DW x-op-code x-endian DI semproc semop)
+    (dai x-basename "w" OP_SIZE_W x-op-code x-endian SI semproc semop)))
 
 (define-pmacro (define-atomic-insns x-endian)
   (begin
-    (dni (.str "xadddw" x-endian)
-         "xadddw"
-         (endian-isas x-endian)
-         (.str "xadddw [$dst" x-endian "+$offset16],$src" x-endian)
-         (+ (f-imm32 0) (.sym src x-endian) (.sym dst x-endian)
-            offset16 OP_MODE_XADD OP_SIZE_DW OP_CLASS_STX)
-         (sem-exchange-and-add x-endian DI)
-         ())
-    (dni (.str "xaddw" x-endian)
-         "xaddw"
-         (endian-isas x-endian)
-         (.str "xaddw [$dst" x-endian "+$offset16],$src" x-endian)
-         (+ (f-imm32 0) (.sym src x-endian) (.sym dst x-endian)
-            offset16 OP_MODE_XADD OP_SIZE_W OP_CLASS_STX)
-         (sem-exchange-and-add x-endian SI)
-         ())))
+    (dais add
+          (.if (.eq x-endian le) OP_ATOMIC_LE_ADD OP_ATOMIC_BE_ADD)
+          x-endian sem-atomic-op add)
+    (dais fadd
+          (.if (.eq x-endian le) OP_ATOMIC_LE_FADD OP_ATOMIC_BE_FADD)
+          x-endian sem-atomic-op-fetch add)
+    (dais or
+          (.if (.eq x-endian le) OP_ATOMIC_LE_OR OP_ATOMIC_BE_OR)
+          x-endian sem-atomic-op or)
+    (dais for
+          (.if (.eq x-endian le) OP_ATOMIC_LE_FOR OP_ATOMIC_BE_FOR)
+          x-endian sem-atomic-op-fetch or)
+    (dais and
+          (.if (.eq x-endian le) OP_ATOMIC_LE_AND OP_ATOMIC_BE_AND)
+          x-endian sem-atomic-op and)
+    (dais fand
+          (.if (.eq x-endian le) OP_ATOMIC_LE_FAND OP_ATOMIC_BE_FAND)
+          x-endian sem-atomic-op-fetch and)
+    (dais xor
+          (.if (.eq x-endian le) OP_ATOMIC_LE_XOR OP_ATOMIC_BE_XOR)
+          x-endian sem-atomic-op xor)
+    (dais fxor
+          (.if (.eq x-endian le) OP_ATOMIC_LE_FXOR OP_ATOMIC_BE_FXOR)
+          x-endian sem-atomic-op-fetch xor)
+    ;; CHG and CMP have only "fetch" variants.
+    (dais chg
+          (.if (.eq x-endian le) OP_ATOMIC_LE_CHG OP_ATOMIC_BE_CHG)
+          x-endian sem-exchange ())
+    (dais cmp
+          (.if (.eq x-endian le) OP_ATOMIC_LE_CMP OP_ATOMIC_BE_CMP)
+          x-endian sem-cmp-exchange ())))
 
 (define-atomic-insns le)
 (define-atomic-insns be)
diff --git a/opcodes/ChangeLog b/opcodes/ChangeLog
index d954bb9..6debcbc 100644
--- a/opcodes/ChangeLog
+++ b/opcodes/ChangeLog
@@ -1,5 +1,12 @@
 2023-05-10  Jose E. Marchesi  <jose.marchesi@oracle.com>
 
+	* bpf-desc.c: Regenerate.
+	* bpf-desc.h: Likewise.
+	* bpf-opc.h: Likewise.
+	* bpf-opc.c: Likewise.
+
+2023-05-10  Jose E. Marchesi  <jose.marchesi@oracle.com>
+
 	* cgen-dis.in (print_insn): Use CGEN_INSN_LGUINT for instruction
 	base values.
 	* cgen-dis.c (cgen_dis_lookup_insn): Likewise.
diff --git a/opcodes/bpf-desc.c b/opcodes/bpf-desc.c
index 7bef587..5e18b0c 100644
--- a/opcodes/bpf-desc.c
+++ b/opcodes/bpf-desc.c
@@ -177,6 +177,7 @@ const CGEN_HW_ENTRY bpf_cgen_hw_table[] =
   { "h-iaddr", HW_H_IADDR, CGEN_ASM_NONE, 0, { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xf0" } } } } },
   { "h-gpr", HW_H_GPR, CGEN_ASM_KEYWORD, & bpf_cgen_opval_h_gpr, { 0, { { { (1<<MACH_BPF)|(1<<MACH_XBPF), 0 } }, { { 1, "\xf0" } } } } },
   { "h-pc", HW_H_PC, CGEN_ASM_NONE, 0, { 0|A(PROFILE)|A(PC), { { { (1<<MACH_BASE), 0 } }, { { 1, "\xf0" } } } } },
+  { "h-r0", HW_H_R0, CGEN_ASM_NONE, 0, { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xf0" } } } } },
   { "h-sint64", HW_H_SINT64, CGEN_ASM_NONE, 0, { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xf0" } } } } },
   { 0, 0, CGEN_ASM_NONE, 0, { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x80" } } } } }
 };
@@ -197,6 +198,7 @@ const CGEN_IFLD bpf_cgen_ifld_table[] =
   { BPF_F_OP_CLASS, "f-op-class", 0, 8, 2, 3, { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xf0" } } } }  },
   { BPF_F_OP_MODE, "f-op-mode", 0, 8, 7, 3, { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xf0" } } } }  },
   { BPF_F_OP_SIZE, "f-op-size", 0, 8, 4, 2, { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xf0" } } } }  },
+  { BPF_F_OP_ATOMIC, "f-op-atomic", 32, 64, 31, 32, { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xf0" } } } }  },
   { BPF_F_DSTLE, "f-dstle", 8, 8, 3, 4, { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }  },
   { BPF_F_SRCLE, "f-srcle", 8, 8, 7, 4, { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }  },
   { BPF_F_DSTBE, "f-dstbe", 8, 8, 7, 4, { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }  },
@@ -1571,6 +1573,96 @@ static const CGEN_IBASE bpf_cgen_insn_table[MAX_INSNS] =
     BPF_INSN_XADDWLE, "xaddwle", "xaddw", 64,
     { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
   },
+/* xfadddw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XFADDDWLE, "xfadddwle", "xfadddw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xfaddw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XFADDWLE, "xfaddwle", "xfaddw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xordw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XORDWLE, "xordwle", "xordw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xorw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XORWLE, "xorwle", "xorw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xfordw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XFORDWLE, "xfordwle", "xfordw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xforw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XFORWLE, "xforwle", "xforw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xanddw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XANDDWLE, "xanddwle", "xanddw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xandw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XANDWLE, "xandwle", "xandw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xfanddw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XFANDDWLE, "xfanddwle", "xfanddw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xfandw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XFANDWLE, "xfandwle", "xfandw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xxordw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XXORDWLE, "xxordwle", "xxordw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xxorw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XXORWLE, "xxorwle", "xxorw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xfxordw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XFXORDWLE, "xfxordwle", "xfxordw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xfxorw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XFXORWLE, "xfxorwle", "xfxorw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xchgdw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XCHGDWLE, "xchgdwle", "xchgdw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xchgw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XCHGWLE, "xchgwle", "xchgw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xcmpdw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XCMPDWLE, "xcmpdwle", "xcmpdw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
+/* xcmpw [$dstle+$offset16],$srcle */
+  {
+    BPF_INSN_XCMPWLE, "xcmpwle", "xcmpw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\xa0" } } } }
+  },
 /* xadddw [$dstbe+$offset16],$srcbe */
   {
     BPF_INSN_XADDDWBE, "xadddwbe", "xadddw", 64,
@@ -1581,6 +1673,96 @@ static const CGEN_IBASE bpf_cgen_insn_table[MAX_INSNS] =
     BPF_INSN_XADDWBE, "xaddwbe", "xaddw", 64,
     { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
   },
+/* xfadddw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XFADDDWBE, "xfadddwbe", "xfadddw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xfaddw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XFADDWBE, "xfaddwbe", "xfaddw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xordw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XORDWBE, "xordwbe", "xordw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xorw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XORWBE, "xorwbe", "xorw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xfordw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XFORDWBE, "xfordwbe", "xfordw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xforw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XFORWBE, "xforwbe", "xforw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xanddw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XANDDWBE, "xanddwbe", "xanddw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xandw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XANDWBE, "xandwbe", "xandw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xfanddw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XFANDDWBE, "xfanddwbe", "xfanddw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xfandw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XFANDWBE, "xfandwbe", "xfandw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xxordw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XXORDWBE, "xxordwbe", "xxordw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xxorw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XXORWBE, "xxorwbe", "xxorw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xfxordw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XFXORDWBE, "xfxordwbe", "xfxordw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xfxorw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XFXORWBE, "xfxorwbe", "xfxorw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xchgdw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XCHGDWBE, "xchgdwbe", "xchgdw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xchgw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XCHGWBE, "xchgwbe", "xchgw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xcmpdw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XCMPDWBE, "xcmpdwbe", "xcmpdw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
+/* xcmpw [$dstbe+$offset16],$srcbe */
+  {
+    BPF_INSN_XCMPWBE, "xcmpwbe", "xcmpw", 64,
+    { 0, { { { (1<<MACH_BASE), 0 } }, { { 1, "\x50" } } } }
+  },
 /* brkpt */
   {
     BPF_INSN_BRKPT, "brkpt", "brkpt", 64,
diff --git a/opcodes/bpf-desc.h b/opcodes/bpf-desc.h
index a4b2076..2ecce2c 100644
--- a/opcodes/bpf-desc.h
+++ b/opcodes/bpf-desc.h
@@ -50,7 +50,7 @@ extern "C" {
 #define CGEN_INT_INSN_P 0
 
 /* Maximum number of syntax elements in an instruction.  */
-#define CGEN_ACTUAL_MAX_SYNTAX_ELEMENTS 16
+#define CGEN_ACTUAL_MAX_SYNTAX_ELEMENTS 17
 
 /* CGEN_MNEMONIC_OPERANDS is defined if mnemonics have operands.
    e.g. In "b,a foo" the ",a" is an operand.  If mnemonics have operands
@@ -85,10 +85,24 @@ typedef enum insn_op_class {
  , OP_CLASS_ALU, OP_CLASS_JMP, OP_CLASS_JMP32, OP_CLASS_ALU64
 } INSN_OP_CLASS;
 
+/* Enum declaration for eBPF atomic insn opcode.  */
+typedef enum insn_atomic_op_le {
+  OP_ATOMIC_LE_ADD = 0, OP_ATOMIC_LE_OR = 64, OP_ATOMIC_LE_AND = 80, OP_ATOMIC_LE_XOR = 160
+ , OP_ATOMIC_LE_FADD = 1, OP_ATOMIC_LE_FOR = 65, OP_ATOMIC_LE_FAND = 81, OP_ATOMIC_LE_FXOR = 161
+ , OP_ATOMIC_LE_CHG = 225, OP_ATOMIC_LE_CMP = 241
+} INSN_ATOMIC_OP_LE;
+
+/* Enum declaration for eBPF atomic insn opcode.  */
+typedef enum insn_atomic_op_be {
+  OP_ATOMIC_BE_ADD = 0, OP_ATOMIC_BE_OR = 1073741824, OP_ATOMIC_BE_AND = 1342177280, OP_ATOMIC_BE_XOR = 0xa0000000
+ , OP_ATOMIC_BE_FADD = 16777216, OP_ATOMIC_BE_FOR = 1090519040, OP_ATOMIC_BE_FAND = 1358954496, OP_ATOMIC_BE_FXOR = 0xa1000000
+ , OP_ATOMIC_BE_CHG = 0xe1000000, OP_ATOMIC_BE_CMP = 0xf1000000
+} INSN_ATOMIC_OP_BE;
+
 /* Enum declaration for eBPF load/store instruction modes.  */
 typedef enum insn_op_mode {
   OP_MODE_IMM = 0, OP_MODE_ABS = 1, OP_MODE_IND = 2, OP_MODE_MEM = 3
- , OP_MODE_XADD = 6
+ , OP_MODE_ATOMIC = 6
 } INSN_OP_MODE;
 
 /* Enum declaration for eBPF load/store instruction sizes.  */
@@ -140,10 +154,10 @@ typedef enum cgen_ifld_attr {
 /* Enum declaration for bpf ifield types.  */
 typedef enum ifield_type {
   BPF_F_NIL, BPF_F_ANYOF, BPF_F_OP_CODE, BPF_F_OP_SRC
- , BPF_F_OP_CLASS, BPF_F_OP_MODE, BPF_F_OP_SIZE, BPF_F_DSTLE
- , BPF_F_SRCLE, BPF_F_DSTBE, BPF_F_SRCBE, BPF_F_REGS
- , BPF_F_OFFSET16, BPF_F_IMM32, BPF_F_IMM64_A, BPF_F_IMM64_B
- , BPF_F_IMM64_C, BPF_F_IMM64, BPF_F_MAX
+ , BPF_F_OP_CLASS, BPF_F_OP_MODE, BPF_F_OP_SIZE, BPF_F_OP_ATOMIC
+ , BPF_F_DSTLE, BPF_F_SRCLE, BPF_F_DSTBE, BPF_F_SRCBE
+ , BPF_F_REGS, BPF_F_OFFSET16, BPF_F_IMM32, BPF_F_IMM64_A
+ , BPF_F_IMM64_B, BPF_F_IMM64_C, BPF_F_IMM64, BPF_F_MAX
 } IFIELD_TYPE;
 
 #define MAX_IFLD ((int) BPF_F_MAX)
@@ -171,8 +185,8 @@ typedef enum cgen_hw_attr {
 /* Enum declaration for bpf hardware types.  */
 typedef enum cgen_hw_type {
   HW_H_MEMORY, HW_H_SINT, HW_H_UINT, HW_H_ADDR
- , HW_H_IADDR, HW_H_GPR, HW_H_PC, HW_H_SINT64
- , HW_MAX
+ , HW_H_IADDR, HW_H_GPR, HW_H_PC, HW_H_R0
+ , HW_H_SINT64, HW_MAX
 } CGEN_HW_TYPE;
 
 #define MAX_HW ((int) HW_MAX)
diff --git a/opcodes/bpf-opc.c b/opcodes/bpf-opc.c
index 1cc06f7..cc69a02 100644
--- a/opcodes/bpf-opc.c
+++ b/opcodes/bpf-opc.c
@@ -149,6 +149,14 @@ static const CGEN_IFMT ifmt_exit ATTRIBUTE_UNUSED = {
   64, 64, 0xffffffffffffffff, { { F (F_IMM32) }, { F (F_OFFSET16) }, { F (F_REGS) }, { F (F_OP_CODE) }, { F (F_OP_SRC) }, { F (F_OP_CLASS) }, { 0 } }
 };
 
+static const CGEN_IFMT ifmt_xadddwle ATTRIBUTE_UNUSED = {
+  64, 64, 0xffffffff000000ff, { { F (F_OP_ATOMIC) }, { F (F_OFFSET16) }, { F (F_SRCLE) }, { F (F_OP_MODE) }, { F (F_OP_SIZE) }, { F (F_DSTLE) }, { F (F_OP_CLASS) }, { 0 } }
+};
+
+static const CGEN_IFMT ifmt_xadddwbe ATTRIBUTE_UNUSED = {
+  64, 64, 0xffffffff000000ff, { { F (F_OP_ATOMIC) }, { F (F_OFFSET16) }, { F (F_DSTBE) }, { F (F_OP_MODE) }, { F (F_OP_SIZE) }, { F (F_SRCBE) }, { F (F_OP_CLASS) }, { 0 } }
+};
+
 #undef F
 
 #define A(a) (1 << CGEN_INSN_##a)
@@ -1680,25 +1688,241 @@ static const CGEN_OPCODE bpf_cgen_insn_opcode_table[MAX_INSNS] =
   {
     { 0, 0, 0, 0 },
     { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
-    & ifmt_ldxwle, { 0xdb }
+    & ifmt_xadddwle, { 0xdb }
   },
 /* xaddw [$dstle+$offset16],$srcle */
   {
     { 0, 0, 0, 0 },
     { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
-    & ifmt_ldxwle, { 0xc3 }
+    & ifmt_xadddwle, { 0xc3 }
+  },
+/* xfadddw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0x1000000db }
+  },
+/* xfaddw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0x1000000c3 }
+  },
+/* xordw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0x40000000db }
+  },
+/* xorw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0x40000000c3 }
+  },
+/* xfordw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0x41000000db }
+  },
+/* xforw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0x41000000c3 }
+  },
+/* xanddw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0x50000000db }
+  },
+/* xandw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0x50000000c3 }
+  },
+/* xfanddw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0x51000000db }
+  },
+/* xfandw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0x51000000c3 }
+  },
+/* xxordw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0xa0000000db }
+  },
+/* xxorw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0xa0000000c3 }
+  },
+/* xfxordw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0xa1000000db }
+  },
+/* xfxorw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0xa1000000c3 }
+  },
+/* xchgdw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0xe1000000db }
+  },
+/* xchgw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0xe1000000c3 }
+  },
+/* xcmpdw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0xf1000000db }
+  },
+/* xcmpw [$dstle+$offset16],$srcle */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTLE), '+', OP (OFFSET16), ']', ',', OP (SRCLE), 0 } },
+    & ifmt_xadddwle, { 0xf1000000c3 }
   },
 /* xadddw [$dstbe+$offset16],$srcbe */
   {
     { 0, 0, 0, 0 },
     { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
-    & ifmt_ldxwbe, { 0xdb }
+    & ifmt_xadddwbe, { 0xdb }
   },
 /* xaddw [$dstbe+$offset16],$srcbe */
   {
     { 0, 0, 0, 0 },
     { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
-    & ifmt_ldxwbe, { 0xc3 }
+    & ifmt_xadddwbe, { 0xc3 }
+  },
+/* xfadddw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0x1000000000000db }
+  },
+/* xfaddw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0x1000000000000c3 }
+  },
+/* xordw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0x40000000000000db }
+  },
+/* xorw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0x40000000000000c3 }
+  },
+/* xfordw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0x41000000000000db }
+  },
+/* xforw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0x41000000000000c3 }
+  },
+/* xanddw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0x50000000000000db }
+  },
+/* xandw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0x50000000000000c3 }
+  },
+/* xfanddw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0x51000000000000db }
+  },
+/* xfandw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0x51000000000000c3 }
+  },
+/* xxordw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0xa0000000000000db }
+  },
+/* xxorw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0xa0000000000000c3 }
+  },
+/* xfxordw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0xa1000000000000db }
+  },
+/* xfxorw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0xa1000000000000c3 }
+  },
+/* xchgdw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0xe1000000000000db }
+  },
+/* xchgw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0xe1000000000000c3 }
+  },
+/* xcmpdw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0xf1000000000000db }
+  },
+/* xcmpw [$dstbe+$offset16],$srcbe */
+  {
+    { 0, 0, 0, 0 },
+    { { MNEM, ' ', '[', OP (DSTBE), '+', OP (OFFSET16), ']', ',', OP (SRCBE), 0 } },
+    & ifmt_xadddwbe, { 0xf1000000000000c3 }
   },
 /* brkpt */
   {
diff --git a/opcodes/bpf-opc.h b/opcodes/bpf-opc.h
index f838afe..b8496b0 100644
--- a/opcodes/bpf-opc.h
+++ b/opcodes/bpf-opc.h
@@ -111,8 +111,17 @@ typedef enum cgen_insn_type {
  , BPF_INSN_JSLTRBE, BPF_INSN_JSLT32IBE, BPF_INSN_JSLT32RBE, BPF_INSN_JSLEIBE
  , BPF_INSN_JSLERBE, BPF_INSN_JSLE32IBE, BPF_INSN_JSLE32RBE, BPF_INSN_CALLLE
  , BPF_INSN_CALLBE, BPF_INSN_CALLRLE, BPF_INSN_CALLRBE, BPF_INSN_JA
- , BPF_INSN_EXIT, BPF_INSN_XADDDWLE, BPF_INSN_XADDWLE, BPF_INSN_XADDDWBE
- , BPF_INSN_XADDWBE, BPF_INSN_BRKPT
+ , BPF_INSN_EXIT, BPF_INSN_XADDDWLE, BPF_INSN_XADDWLE, BPF_INSN_XFADDDWLE
+ , BPF_INSN_XFADDWLE, BPF_INSN_XORDWLE, BPF_INSN_XORWLE, BPF_INSN_XFORDWLE
+ , BPF_INSN_XFORWLE, BPF_INSN_XANDDWLE, BPF_INSN_XANDWLE, BPF_INSN_XFANDDWLE
+ , BPF_INSN_XFANDWLE, BPF_INSN_XXORDWLE, BPF_INSN_XXORWLE, BPF_INSN_XFXORDWLE
+ , BPF_INSN_XFXORWLE, BPF_INSN_XCHGDWLE, BPF_INSN_XCHGWLE, BPF_INSN_XCMPDWLE
+ , BPF_INSN_XCMPWLE, BPF_INSN_XADDDWBE, BPF_INSN_XADDWBE, BPF_INSN_XFADDDWBE
+ , BPF_INSN_XFADDWBE, BPF_INSN_XORDWBE, BPF_INSN_XORWBE, BPF_INSN_XFORDWBE
+ , BPF_INSN_XFORWBE, BPF_INSN_XANDDWBE, BPF_INSN_XANDWBE, BPF_INSN_XFANDDWBE
+ , BPF_INSN_XFANDWBE, BPF_INSN_XXORDWBE, BPF_INSN_XXORWBE, BPF_INSN_XFXORDWBE
+ , BPF_INSN_XFXORWBE, BPF_INSN_XCHGDWBE, BPF_INSN_XCHGWBE, BPF_INSN_XCMPDWBE
+ , BPF_INSN_XCMPWBE, BPF_INSN_BRKPT
 } CGEN_INSN_TYPE;
 
 /* Index of `invalid' insn place holder.  */
@@ -132,6 +141,7 @@ struct cgen_fields
   long f_op_class;
   long f_op_mode;
   long f_op_size;
+  long f_op_atomic;
   long f_dstle;
   long f_srcle;
   long f_dstbe;