1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
|
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 2
; RUN: opt < %s -passes=memcpyopt -verify-memoryssa -S | FileCheck %s
; Test that stack-move optimization works when src is a GEP into an alloca.
; For the optimization to trigger:
; - The copy must cover the entire dest alloca (Size == DestSize, DestOffset == 0)
; - SrcOffset must be a multiple of DestAlloca's alignment
; - SrcOffset must be non-negative
declare void @llvm.memcpy.p0.p0.i64(ptr noalias nocapture writeonly, ptr noalias nocapture readonly, i64, i1 immarg)
declare void @llvm.lifetime.start.p0(ptr nocapture)
declare void @llvm.lifetime.end.p0(ptr nocapture)
declare void @use_nocapture(ptr nocapture)
; Basic test: memcpy from GEP(src) to dest alloca
; src = [16 x i8], dest = [8 x i8] align 8, copy 8 bytes
; SrcOffset(8) is a multiple of DestAlign(8), so optimization applies.
; After optimization: dest uses become src+8
define void @memcpy_src_gep_to_dest_alloca() {
; CHECK-LABEL: define void @memcpy_src_gep_to_dest_alloca() {
; CHECK-NEXT: [[SRC:%.*]] = alloca [16 x i8], align 8
; CHECK-NEXT: [[SRC_GEP:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 8
; CHECK-NEXT: [[SRC_GEP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 8
; CHECK-NEXT: store i64 42, ptr [[SRC_GEP1]], align 4
; CHECK-NEXT: call void @use_nocapture(ptr captures(none) [[SRC_GEP1]])
; CHECK-NEXT: call void @use_nocapture(ptr captures(none) [[SRC_GEP]])
; CHECK-NEXT: ret void
;
%src = alloca [16 x i8], align 4
%dest = alloca [8 x i8], align 8
call void @llvm.lifetime.start.p0(ptr %src)
call void @llvm.lifetime.start.p0(ptr %dest)
%src.gep = getelementptr inbounds i8, ptr %src, i64 8
store i64 42, ptr %src.gep
call void @use_nocapture(ptr nocapture %src.gep)
call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dest, ptr align 8 %src.gep, i64 8, i1 false)
call void @use_nocapture(ptr nocapture %dest)
call void @llvm.lifetime.end.p0(ptr %src)
call void @llvm.lifetime.end.p0(ptr %dest)
ret void
}
; Test: memcpy from GEP(src) to dest alloca with different offset
; src = [12 x i8], dest = [8 x i8] align 4, copy 8 bytes from src+4
; SrcOffset(4) is a multiple of DestAlign(4), so optimization applies.
; After optimization: dest uses become src+4
define void @memcpy_src_gep_offset4_to_dest_alloca() {
; CHECK-LABEL: define void @memcpy_src_gep_offset4_to_dest_alloca() {
; CHECK-NEXT: [[SRC:%.*]] = alloca [12 x i8], align 4
; CHECK-NEXT: [[SRC_GEP:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
; CHECK-NEXT: [[SRC_GEP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
; CHECK-NEXT: store i64 42, ptr [[SRC_GEP1]], align 4
; CHECK-NEXT: call void @use_nocapture(ptr captures(none) [[SRC_GEP1]])
; CHECK-NEXT: call void @use_nocapture(ptr captures(none) [[SRC_GEP]])
; CHECK-NEXT: ret void
;
%src = alloca [12 x i8], align 4
%dest = alloca [8 x i8], align 4
call void @llvm.lifetime.start.p0(ptr %src)
call void @llvm.lifetime.start.p0(ptr %dest)
%src.gep = getelementptr inbounds i8, ptr %src, i64 4
store i64 42, ptr %src.gep
call void @use_nocapture(ptr nocapture %src.gep)
call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src.gep, i64 8, i1 false)
call void @use_nocapture(ptr nocapture %dest)
call void @llvm.lifetime.end.p0(ptr %src)
call void @llvm.lifetime.end.p0(ptr %dest)
ret void
}
; Test: load/store from GEP(src) to dest alloca
; src = [8 x i8], dest = [4 x i8] align 4, load/store 4 bytes from src+4
; SrcOffset(4) is a multiple of DestAlign(4), so optimization applies.
define void @load_store_src_gep_to_dest_alloca() {
; CHECK-LABEL: define void @load_store_src_gep_to_dest_alloca() {
; CHECK-NEXT: [[SRC:%.*]] = alloca [8 x i8], align 4
; CHECK-NEXT: [[SRC_GEP:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
; CHECK-NEXT: [[SRC_GEP1:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
; CHECK-NEXT: store i32 42, ptr [[SRC_GEP1]], align 4
; CHECK-NEXT: call void @use_nocapture(ptr captures(none) [[SRC_GEP1]])
; CHECK-NEXT: call void @use_nocapture(ptr captures(none) [[SRC_GEP]])
; CHECK-NEXT: ret void
;
%src = alloca [8 x i8], align 4
%dest = alloca [4 x i8], align 4
call void @llvm.lifetime.start.p0(ptr %src)
call void @llvm.lifetime.start.p0(ptr %dest)
%src.gep = getelementptr inbounds i8, ptr %src, i64 4
store i32 42, ptr %src.gep
call void @use_nocapture(ptr nocapture %src.gep)
%val = load i32, ptr %src.gep
store i32 %val, ptr %dest
call void @use_nocapture(ptr nocapture %dest)
call void @llvm.lifetime.end.p0(ptr %src)
call void @llvm.lifetime.end.p0(ptr %dest)
ret void
}
; Test: both src and dest are direct allocas (no offset), same size
; This is the basic stack-move case, included here for completeness.
define void @memcpy_both_direct_allocas() {
; CHECK-LABEL: define void @memcpy_both_direct_allocas() {
; CHECK-NEXT: [[SRC:%.*]] = alloca [8 x i8], align 8
; CHECK-NEXT: store i64 42, ptr [[SRC]], align 4
; CHECK-NEXT: call void @use_nocapture(ptr captures(none) [[SRC]])
; CHECK-NEXT: call void @use_nocapture(ptr captures(none) [[SRC]])
; CHECK-NEXT: ret void
;
%src = alloca [8 x i8], align 4
%dest = alloca [8 x i8], align 8
call void @llvm.lifetime.start.p0(ptr %src)
call void @llvm.lifetime.start.p0(ptr %dest)
store i64 42, ptr %src
call void @use_nocapture(ptr nocapture %src)
call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dest, ptr align 4 %src, i64 8, i1 false)
call void @use_nocapture(ptr nocapture %dest)
call void @llvm.lifetime.end.p0(ptr %src)
call void @llvm.lifetime.end.p0(ptr %dest)
ret void
}
; Negative test: dest has offset (dest is GEP, not direct alloca)
; The optimization requires DestOffset == 0.
define void @no_optimize_dest_has_offset() {
; CHECK-LABEL: define void @no_optimize_dest_has_offset() {
; CHECK-NEXT: [[SRC:%.*]] = alloca [16 x i8], align 4
; CHECK-NEXT: [[DEST:%.*]] = alloca [16 x i8], align 8
; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[SRC]])
; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[DEST]])
; CHECK-NEXT: [[SRC_GEP:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 8
; CHECK-NEXT: [[DEST_GEP:%.*]] = getelementptr inbounds i8, ptr [[DEST]], i64 8
; CHECK-NEXT: store i64 42, ptr [[SRC_GEP]], align 4
; CHECK-NEXT: call void @use_nocapture(ptr captures(none) [[SRC_GEP]])
; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST_GEP]], ptr align 4 [[SRC_GEP]], i64 8, i1 false)
; CHECK-NEXT: call void @use_nocapture(ptr captures(none) [[DEST_GEP]])
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[SRC]])
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[DEST]])
; CHECK-NEXT: ret void
;
%src = alloca [16 x i8], align 4
%dest = alloca [16 x i8], align 8
call void @llvm.lifetime.start.p0(ptr %src)
call void @llvm.lifetime.start.p0(ptr %dest)
%src.gep = getelementptr inbounds i8, ptr %src, i64 8
%dest.gep = getelementptr inbounds i8, ptr %dest, i64 8
store i64 42, ptr %src.gep
call void @use_nocapture(ptr nocapture %src.gep)
call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest.gep, ptr align 4 %src.gep, i64 8, i1 false)
call void @use_nocapture(ptr nocapture %dest.gep)
call void @llvm.lifetime.end.p0(ptr %src)
call void @llvm.lifetime.end.p0(ptr %dest)
ret void
}
; Negative test: copy doesn't cover entire dest alloca (Size != DestSize)
; src = [12 x i8], dest = [16 x i8], copy only 8 bytes
define void @no_optimize_partial_dest_copy() {
; CHECK-LABEL: define void @no_optimize_partial_dest_copy() {
; CHECK-NEXT: [[SRC:%.*]] = alloca [12 x i8], align 4
; CHECK-NEXT: [[DEST:%.*]] = alloca [16 x i8], align 4
; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[SRC]])
; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[DEST]])
; CHECK-NEXT: [[SRC_GEP:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
; CHECK-NEXT: store i64 42, ptr [[SRC_GEP]], align 4
; CHECK-NEXT: call void @use_nocapture(ptr captures(none) [[SRC_GEP]])
; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 4 [[DEST]], ptr align 4 [[SRC_GEP]], i64 8, i1 false)
; CHECK-NEXT: call void @use_nocapture(ptr captures(none) [[DEST]])
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[SRC]])
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[DEST]])
; CHECK-NEXT: ret void
;
%src = alloca [12 x i8], align 4
%dest = alloca [16 x i8], align 4
call void @llvm.lifetime.start.p0(ptr %src)
call void @llvm.lifetime.start.p0(ptr %dest)
%src.gep = getelementptr inbounds i8, ptr %src, i64 4
store i64 42, ptr %src.gep
call void @use_nocapture(ptr nocapture %src.gep)
call void @llvm.memcpy.p0.p0.i64(ptr align 4 %dest, ptr align 4 %src.gep, i64 8, i1 false)
call void @use_nocapture(ptr nocapture %dest)
call void @llvm.lifetime.end.p0(ptr %src)
call void @llvm.lifetime.end.p0(ptr %dest)
ret void
}
; Negative test: SrcOffset not a multiple of DestAlign
; src = [12 x i8] with offset 4, dest = [8 x i8] align 8
; SrcOffset(4) % DestAlign(8) = 4 != 0 -> rejected
define void @no_optimize_alignment_mismatch() {
; CHECK-LABEL: define void @no_optimize_alignment_mismatch() {
; CHECK-NEXT: [[SRC:%.*]] = alloca [12 x i8], align 4
; CHECK-NEXT: [[DEST:%.*]] = alloca [8 x i8], align 8
; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[SRC]])
; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[DEST]])
; CHECK-NEXT: [[SRC_GEP:%.*]] = getelementptr inbounds i8, ptr [[SRC]], i64 4
; CHECK-NEXT: store i64 42, ptr [[SRC_GEP]], align 4
; CHECK-NEXT: call void @use_nocapture(ptr captures(none) [[SRC_GEP]])
; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DEST]], ptr align 4 [[SRC_GEP]], i64 8, i1 false)
; CHECK-NEXT: call void @use_nocapture(ptr captures(none) [[DEST]])
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[SRC]])
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[DEST]])
; CHECK-NEXT: ret void
;
%src = alloca [12 x i8], align 4
%dest = alloca [8 x i8], align 8
call void @llvm.lifetime.start.p0(ptr %src)
call void @llvm.lifetime.start.p0(ptr %dest)
%src.gep = getelementptr inbounds i8, ptr %src, i64 4
store i64 42, ptr %src.gep
call void @use_nocapture(ptr nocapture %src.gep)
call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dest, ptr align 4 %src.gep, i64 8, i1 false)
call void @use_nocapture(ptr nocapture %dest)
call void @llvm.lifetime.end.p0(ptr %src)
call void @llvm.lifetime.end.p0(ptr %dest)
ret void
}
; Negative test: Clobbering store to source offset between memcpy from source and memcpy to dest
; This is a minimal reproducer for the bug in LLVM PR #176436.
define void @no_optimize_clobbering_store_to_src_offset(ptr noalias %dst) {
; CHECK-LABEL: define void @no_optimize_clobbering_store_to_src_offset
; CHECK-SAME: (ptr noalias [[DST:%.*]]) {
; CHECK-NEXT: [[TEMP2:%.*]] = alloca { i64, i64 }, align 8
; CHECK-NEXT: [[TEMP1:%.*]] = alloca { i64, i64 }, align 8
; CHECK-NEXT: [[LOCAL:%.*]] = alloca { [48 x i8], { i64, i64 }, ptr }, align 8
; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[TEMP1]])
; CHECK-NEXT: [[DST_BUF:%.*]] = getelementptr i8, ptr [[DST]], i64 48
; CHECK-NEXT: [[LOCAL_BUF:%.*]] = getelementptr inbounds i8, ptr [[LOCAL]], i64 48
; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[LOCAL_BUF]], ptr align 8 [[DST_BUF]], i64 16, i1 false)
; CHECK-NEXT: store i8 0, ptr [[DST_BUF]], align 1
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[TEMP1]])
; CHECK-NEXT: call void @llvm.lifetime.start.p0(ptr [[TEMP2]])
; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[TEMP2]], ptr align 8 [[LOCAL_BUF]], i64 16, i1 false)
; CHECK-NEXT: store i8 0, ptr [[LOCAL_BUF]], align 1
; CHECK-NEXT: call void @llvm.memcpy.p0.p0.i64(ptr align 8 [[DST_BUF]], ptr align 8 [[TEMP2]], i64 16, i1 false)
; CHECK-NEXT: call void @llvm.lifetime.end.p0(ptr [[TEMP2]])
; CHECK-NEXT: ret void
;
%temp2 = alloca { i64, i64 }, align 8
%temp1 = alloca { i64, i64 }, align 8
%local = alloca { [48 x i8], { i64, i64 }, ptr }, align 8
; First move: copy from dst+48 to local+48 via temp1
call void @llvm.lifetime.start.p0(ptr %temp1)
%dst_buf = getelementptr i8, ptr %dst, i64 48
call void @llvm.memcpy.p0.p0.i64(ptr align 8 %temp1, ptr align 8 %dst_buf, i64 16, i1 false)
store i8 0, ptr %dst_buf, align 1
%local_buf = getelementptr inbounds i8, ptr %local, i64 48
call void @llvm.memcpy.p0.p0.i64(ptr align 8 %local_buf, ptr align 8 %temp1, i64 16, i1 false)
call void @llvm.lifetime.end.p0(ptr %temp1)
; Second move: copy from local+48 back to dst+48 via temp2
; BUG: PR incorrectly eliminated temp2 but the store below clobbers part of local_buf first!
; The fix ensures we check the right portion of SrcAlloca for any clobbering.
call void @llvm.lifetime.start.p0(ptr %temp2)
call void @llvm.memcpy.p0.p0.i64(ptr align 8 %temp2, ptr align 8 %local_buf, i64 16, i1 false)
store i8 0, ptr %local_buf, align 1 ; <-- clobbers byte 48 of %local
call void @llvm.memcpy.p0.p0.i64(ptr align 8 %dst_buf, ptr align 8 %temp2, i64 16, i1 false)
call void @llvm.lifetime.end.p0(ptr %temp2)
ret void
}
|
