target/i386: fix fisttpl, fisttpll handling of out-of-range values

The fist / fistt family of instructions should all store the most
negative integer in the destination format when the rounded /
truncated integer result is out of range or the input is an invalid
encoding, infinity or NaN.  The fisttpl and fisttpll implementations
(32-bit and 64-bit results, truncate towards zero) failed to do this,
producing the most positive integer in some cases instead.  Fix this
by copying the code used to handle this issue for fistpl and fistpll,
adjusted to use the _round_to_zero functions for the actual
conversion (but without any other changes to that code).

Signed-off-by: Joseph Myers <joseph@codesourcery.com>
Message-Id: <alpine.DEB.2.21.2005152119160.3469@digraph.polyomino.org.uk>
Signed-off-by: Paolo Bonzini <pbonzini@redhat.com>

1 files changed, 100 insertions, 0 deletions

diff --git a/tests/tcg/i386/test-i386-fisttp.c b/tests/tcg/i386/test-i386-fisttp.c
new file mode 100644
index 0000000..16af59a
--- /dev/null
+++ b/tests/tcg/i386/test-i386-fisttp.c
@@ -0,0 +1,100 @@
+/* Test fisttpl and fisttpll instructions.  */
+
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+
+union u {
+    struct { uint64_t sig; uint16_t sign_exp; } s;
+    long double ld;
+};
+
+volatile union u ld_invalid_1 = { .s = { 1, 1234 } };
+
+int main(void)
+{
+    int ret = 0;
+    int32_t res_32;
+    int64_t res_64;
+    __asm__ volatile ("fisttpl %0" : "=m" (res_32) : "t" (0x1p100L) : "st");
+    if (res_32 != INT32_MIN) {
+        printf("FAIL: fisttpl 0x1p100\n");
+        ret = 1;
+    }
+    __asm__ volatile ("fisttpl %0" : "=m" (res_32) : "t" (-0x1p100L) : "st");
+    if (res_32 != INT32_MIN) {
+        printf("FAIL: fisttpl -0x1p100\n");
+        ret = 1;
+    }
+    __asm__ volatile ("fisttpl %0" : "=m" (res_32) : "t" (__builtin_infl()) :
+                      "st");
+    if (res_32 != INT32_MIN) {
+        printf("FAIL: fisttpl inf\n");
+        ret = 1;
+    }
+    __asm__ volatile ("fisttpl %0" : "=m" (res_32) : "t" (-__builtin_infl()) :
+                      "st");
+    if (res_32 != INT32_MIN) {
+        printf("FAIL: fisttpl -inf\n");
+        ret = 1;
+    }
+    __asm__ volatile ("fisttpl %0" : "=m" (res_32) : "t" (__builtin_nanl("")) :
+                      "st");
+    if (res_32 != INT32_MIN) {
+        printf("FAIL: fisttpl nan\n");
+        ret = 1;
+    }
+    __asm__ volatile ("fisttpl %0" : "=m" (res_32) :
+                      "t" (-__builtin_nanl("")) : "st");
+    if (res_32 != INT32_MIN) {
+        printf("FAIL: fisttpl -nan\n");
+        ret = 1;
+    }
+    __asm__ volatile ("fisttpl %0" : "=m" (res_32) : "t" (ld_invalid_1.ld) :
+                      "st");
+    if (res_32 != INT32_MIN) {
+        printf("FAIL: fisttpl invalid\n");
+        ret = 1;
+    }
+    __asm__ volatile ("fisttpll %0" : "=m" (res_64) : "t" (0x1p100L) : "st");
+    if (res_64 != INT64_MIN) {
+        printf("FAIL: fisttpll 0x1p100\n");
+        ret = 1;
+    }
+    __asm__ volatile ("fisttpll %0" : "=m" (res_64) : "t" (-0x1p100L) : "st");
+    if (res_64 != INT64_MIN) {
+        printf("FAIL: fisttpll -0x1p100\n");
+        ret = 1;
+    }
+    __asm__ volatile ("fisttpll %0" : "=m" (res_64) : "t" (__builtin_infl()) :
+                      "st");
+    if (res_64 != INT64_MIN) {
+        printf("FAIL: fisttpll inf\n");
+        ret = 1;
+    }
+    __asm__ volatile ("fisttpll %0" : "=m" (res_64) : "t" (-__builtin_infl()) :
+                      "st");
+    if (res_64 != INT64_MIN) {
+        printf("FAIL: fisttpll -inf\n");
+        ret = 1;
+    }
+    __asm__ volatile ("fisttpll %0" : "=m" (res_64) :
+                      "t" (__builtin_nanl("")) : "st");
+    if (res_64 != INT64_MIN) {
+        printf("FAIL: fisttpll nan\n");
+        ret = 1;
+    }
+    __asm__ volatile ("fisttpll %0" : "=m" (res_64) :
+                      "t" (-__builtin_nanl("")) : "st");
+    if (res_64 != INT64_MIN) {
+        printf("FAIL: fisttpll -nan\n");
+        ret = 1;
+    }
+    __asm__ volatile ("fisttpll %0" : "=m" (res_64) : "t" (ld_invalid_1.ld) :
+                      "st");
+    if (res_64 != INT64_MIN) {
+        printf("FAIL: fisttpll invalid\n");
+        ret = 1;
+    }
+    return ret;
+}