1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
|
/* ===--- amxmovrstransposeintrin.h - AMX_MOVRS_TRANSPOSE intrinsics --------===
*
* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
* See https://llvm.org/LICENSE.txt for license information.
* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
*
* ===-----------------------------------------------------------------------===
*/
#ifndef __IMMINTRIN_H
#error \
"Never use <amxmovrstransposeintrin.h> directly; use <immintrin.h> instead."
#endif /* __IMMINTRIN_H */
#ifndef __AMX_MOVRS_TRANSPOSEINTRIN_H
#define __AMX_MOVRS_TRANSPOSEINTRIN_H
#ifdef __x86_64__
#define __DEFAULT_FN_ATTRS \
__attribute__((__always_inline__, __nodebug__, \
__target__("amx-transpose,amx-movrs")))
#define _tile_2rpntlvwz0rs(tdst, base, stride) \
__builtin_ia32_t2rpntlvwz0rs(tdst, base, stride)
#define _tile_2rpntlvwz0rst1(tdst, base, stride) \
__builtin_ia32_t2rpntlvwz0rst1(tdst, base, stride)
#define _tile_2rpntlvwz1rs(tdst, base, stride) \
__builtin_ia32_t2rpntlvwz1rs(tdst, base, stride)
#define _tile_2rpntlvwz1rst1(tdst, base, stride) \
__builtin_ia32_t2rpntlvwz1rst1(tdst, base, stride)
static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz0rs_internal(
unsigned short row, unsigned short col0, unsigned short col1,
_tile1024i *dst0, _tile1024i *dst1, const void *base,
__SIZE_TYPE__ stride) {
// Use __tile1024i_1024a* to escape the alignment check in
// clang/test/Headers/x86-intrinsics-headers-clean.cpp
__builtin_ia32_t2rpntlvwz0rs_internal(
row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
(__SIZE_TYPE__)(stride));
}
static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz0rst1_internal(
unsigned short row, unsigned short col0, unsigned short col1,
_tile1024i *dst0, _tile1024i *dst1, const void *base,
__SIZE_TYPE__ stride) {
__builtin_ia32_t2rpntlvwz0rst1_internal(
row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
(__SIZE_TYPE__)(stride));
}
static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz1rs_internal(
unsigned short row, unsigned short col0, unsigned short col1,
_tile1024i *dst0, _tile1024i *dst1, const void *base,
__SIZE_TYPE__ stride) {
__builtin_ia32_t2rpntlvwz1rs_internal(
row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
(__SIZE_TYPE__)(stride));
}
static __inline__ void __DEFAULT_FN_ATTRS _tile_2rpntlvwz1rst1_internal(
unsigned short row, unsigned short col0, unsigned short col1,
_tile1024i *dst0, _tile1024i *dst1, const void *base,
__SIZE_TYPE__ stride) {
__builtin_ia32_t2rpntlvwz1rst1_internal(
row, col0, col1, (_tile1024i_1024a *)dst0, (_tile1024i_1024a *)dst1, base,
(__SIZE_TYPE__)(stride));
}
/// Converts a pair of tiles from memory into VNNI format, and places the
/// results in a pair of destinations specified by dst. The pair of tiles
/// in memory is specified via a tsib; the second tile is after the first
/// one, separated by the same stride that separates each row.
/// The tile configuration for the destination tiles indicates the amount
/// of data to read from memory. The instruction will load a number of rows
/// that is equal to twice the number of rows in tmm1. The size of each row
/// is equal to the average width of the destination tiles. If the second
/// tile is configured with zero rows and columns, only the first tile will
/// be written.
/// Provides a hint to the implementation that the data will likely become
/// read shared in the near future and the data caching can be optimized.
///
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> T2RPNTLVWZ0RS </c> instruction.
///
/// \param dst0
/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
/// \param dst1
/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
/// \param base
/// A pointer to base address.
/// \param stride
/// The stride between the rows' data to be loaded in memory.
__DEFAULT_FN_ATTRS
static void __tile_2rpntlvwz0rs(__tile1024i *dst0, __tile1024i *dst1,
const void *base, __SIZE_TYPE__ stride) {
_tile_2rpntlvwz0rs_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
&dst1->tile, base, stride);
}
/// Converts a pair of tiles from memory into VNNI format, and places the
/// results in a pair of destinations specified by dst. The pair of tiles
/// in memory is specified via a tsib; the second tile is after the first
/// one, separated by the same stride that separates each row.
/// The tile configuration for the destination tiles indicates the amount
/// of data to read from memory. The instruction will load a number of rows
/// that is equal to twice the number of rows in tmm1. The size of each row
/// is equal to the average width of the destination tiles. If the second
/// tile is configured with zero rows and columns, only the first tile will
/// be written.
///
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> T2RPNTLVWZ0T1RS </c> instruction.
///
/// \param dst0
/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
/// \param dst1
/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
/// \param base
/// A pointer to base address.
/// \param stride
/// The stride between the rows' data to be loaded in memory.
__DEFAULT_FN_ATTRS
static void __tile_2rpntlvwz0rst1(__tile1024i *dst0, __tile1024i *dst1,
const void *base, __SIZE_TYPE__ stride) {
_tile_2rpntlvwz0rst1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
&dst1->tile, base, stride);
}
/// Converts a pair of tiles from memory into VNNI format, and places the
/// results in a pair of destinations specified by dst. The pair of tiles
/// in memory is specified via a tsib; the second tile is after the first
/// one, separated by the same stride that separates each row.
/// The tile configuration for the destination tiles indicates the amount
/// of data to read from memory. The instruction will load a number of rows
/// that is equal to twice the number of rows in tmm1. The size of each row
/// is equal to the average width of the destination tiles. If the second
/// tile is configured with zero rows and columns, only the first tile will
/// be written. The last row will be not be read from memory but instead
/// filled with zeros.
/// Provides a hint to the implementation that the data will likely become
/// read shared in the near future and the data caching can be optimized.
///
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> T2RPNTLVWZ1 </c> instruction.
///
/// \param dst0
/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
/// \param dst1
/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
/// \param base
/// A pointer to base address.
/// \param stride
/// The stride between the rows' data to be loaded in memory.
__DEFAULT_FN_ATTRS
static void __tile_2rpntlvwz1rs(__tile1024i *dst0, __tile1024i *dst1,
const void *base, __SIZE_TYPE__ stride) {
_tile_2rpntlvwz1rs_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
&dst1->tile, base, stride);
}
/// Converts a pair of tiles from memory into VNNI format, and places the
/// results in a pair of destinations specified by dst. The pair of tiles
/// in memory is specified via a tsib; the second tile is after the first
/// one, separated by the same stride that separates each row.
/// The tile configuration for the destination tiles indicates the amount
/// of data to read from memory. The instruction will load a number of rows
/// that is equal to twice the number of rows in tmm1. The size of each row
/// is equal to the average width of the destination tiles. If the second
/// tile is configured with zero rows and columns, only the first tile will
/// be written. The last row will be not be read from memory but instead
/// filled with zeros.
/// Provides a hint to the implementation that the data will likely become
/// read shared in the near future and the data caching can be optimized.
///
/// \headerfile <immintrin.h>
///
/// This intrinsic corresponds to the <c> T2RPNTLVWZ1T1RS </c> instruction.
///
/// \param dst0
/// First tile of destination tile pair. Max size is 1024i*2 Bytes.
/// \param dst1
/// Second tile of destination tile pair. Max size is 1024i*2 Bytes.
/// \param base
/// A pointer to base address.
/// \param stride
/// The stride between the rows' data to be loaded in memory.
__DEFAULT_FN_ATTRS
static void __tile_2rpntlvwz1rst1(__tile1024i *dst0, __tile1024i *dst1,
const void *base, __SIZE_TYPE__ stride) {
_tile_2rpntlvwz1rst1_internal(dst0->row, dst0->col, dst1->col, &dst0->tile,
&dst1->tile, base, stride);
}
#undef __DEFAULT_FN_ATTRS
#endif /* __x86_64__ */
#endif /* __AMX_MOVRS_TRANSPOSEINTRIN_H */
|
