1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
|
/* Copyright (C) 2014-2015 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#include <sysdep.h>
/* Implements the functions
char * [r3] strncpy (char *dst [r3], const char *src [r4], size_t n [r5])
AND
char * [r3] stpncpy (char *dst [r3], const char *src [r4], size_t n [r5])
The algorithm is as follows:
> if src and dest are 8 byte aligned, perform double word copy
else
> copy byte by byte on unaligned addresses.
The aligned comparison are made using cmpb instructions. */
/* The focus on optimization for performance improvements are as follows:
1. data alignment [gain from aligned memory access on read/write]
2. POWER7 gains performance with loop unrolling/unwinding
[gain by reduction of branch penalty].
3. The final pad with null bytes is done by calling an optimized
memset. */
#ifdef USE_AS_STPNCPY
# define FUNC_NAME __stpncpy
#else
# define FUNC_NAME strncpy
#endif
#define FRAMESIZE (FRAME_MIN_SIZE+32)
#ifndef MEMSET
/* For builds with no IFUNC support, local calls should be made to internal
GLIBC symbol (created by libc_hidden_builtin_def). */
# ifdef SHARED
# define MEMSET __GI_memset
# else
# define MEMSET memset
# endif
#endif
.machine power7
EALIGN(FUNC_NAME, 4, 0)
CALL_MCOUNT 3
mflr r0 /* load link register LR to r0 */
or r10, r3, r4 /* to verify source and destination */
rldicl. r8, r10, 0, 61 /* is double word aligned .. ? */
std r19, -8(r1) /* save callers register , r19 */
std r18, -16(r1) /* save callers register , r18 */
std r0, 16(r1) /* store the link register */
stdu r1, -FRAMESIZE(r1) /* create the stack frame */
mr r9, r3 /* save r3 into r9 for use */
mr r18, r3 /* save r3 for retCode of strncpy */
bne 0, L(byte_by_byte)
srdi r11, r5, 3 /* compute count for CTR ; count = n/8 */
cmpldi cr7, r11, 3 /* if count > 4 ; perform unrolling 4 times */
ble 7, L(update1)
ld r10, 0(r4) /* load doubleWord from src */
cmpb r8, r10, r8 /* compare src with NULL ,we read just now */
cmpdi cr7, r8, 0 /* if cmpb returned NULL ; we continue */
bne cr7, L(update3)
std r10, 0(r3) /* copy doubleword at offset=0 */
ld r10, 8(r4) /* load next doubleword from offset=8 */
cmpb r8, r10, r8 /* compare src with NULL , we read just now */
cmpdi cr7, r8, 0 /* if cmpb returned NULL ; we continue */
bne 7,L(HopBy8)
addi r8, r11, -4
mr r7, r3
srdi r8, r8, 2
mr r6, r4
addi r8, r8, 1
li r12, 0
mtctr r8
b L(dwordCopy)
.p2align 4
L(dWordUnroll):
std r8, 16(r9)
ld r8, 24(r4) /* load dword,perform loop unrolling again */
cmpb r10, r8, r10
cmpdi cr7, r10, 0
bne cr7, L(HopBy24)
std r8, 24(r7) /* copy dword at offset=24 */
addi r9, r9, 32
addi r4, r4, 32
bdz L(leftDwords) /* continue with loop on counter */
ld r3, 32(r6)
cmpb r8, r3, r10
cmpdi cr7, r8, 0
bne cr7, L(update2)
std r3, 32(r7)
ld r10, 40(r6)
cmpb r8, r10, r8
cmpdi cr7, r8, 0
bne cr7, L(HopBy40)
mr r6, r4 /* update values */
mr r7, r9
mr r11, r0
mr r5, r19
L(dwordCopy):
std r10, 8(r9) /* copy dword at offset=8 */
addi r19, r5, -32
addi r0, r11, -4
ld r8, 16(r4)
cmpb r10, r8, r12
cmpdi cr7, r10, 0
beq cr7, L(dWordUnroll)
addi r9, r9, 16 /* increment dst by 16 */
addi r4, r4, 16 /* increment src by 16 */
addi r5, r5, -16 /* decrement length 'n' by 16 */
addi r0, r11, -2 /* decrement loop counter */
L(dWordUnrollOFF):
ld r10, 0(r4) /* load first dword */
li r8, 0 /* load mask */
cmpb r8, r10, r8
cmpdi cr7, r8, 0
bne cr7, L(byte_by_byte)
mtctr r0
li r7, 0
b L(CopyDword)
.p2align 4
L(loadDWordandCompare):
ld r10, 0(r4)
cmpb r8, r10, r7
cmpdi cr7, r8, 0
bne cr7, L(byte_by_byte)
L(CopyDword):
addi r9, r9, 8
std r10, -8(r9)
addi r4, r4, 8
addi r5, r5, -8
bdnz L(loadDWordandCompare)
L(byte_by_byte):
cmpldi cr7, r5, 3
ble cr7, L(verifyByte)
srdi r10, r5, 2
mr r19, r9
mtctr r10
b L(firstByteUnroll)
.p2align 4
L(bytes_unroll):
lbz r10, 1(r4) /* load byte from src */
cmpdi cr7, r10, 0 /* compare for NULL */
stb r10, 1(r19) /* store byte to dst */
beq cr7, L(updtDestComputeN2ndByte)
addi r4, r4, 4 /* advance src */
lbz r10, -2(r4) /* perform loop unrolling for byte r/w */
cmpdi cr7, r10, 0
stb r10, 2(r19)
beq cr7, L(updtDestComputeN3rdByte)
lbz r10, -1(r4) /* perform loop unrolling for byte r/w */
addi r19, r19, 4
cmpdi cr7, r10, 0
stb r10, -1(r19)
beq cr7, L(ComputeNByte)
bdz L(update0)
L(firstByteUnroll):
lbz r10, 0(r4) /* perform loop unrolling for byte r/w */
cmpdi cr7, 10, 0
stb r10, 0(r19)
bne cr7, L(bytes_unroll)
addi r19, r19, 1
L(ComputeNByte):
subf r9, r19, r9 /* compute 'n'n bytes to fill */
add r8, r9, r5
L(zeroFill):
cmpdi cr7, r8, 0 /* compare if length is zero */
beq cr7, L(update3return)
mr r3, r19 /* fill buffer with */
li r4, 0 /* zero fill buffer */
mr r5, r8 /* how many bytes to fill buffer with */
bl MEMSET /* call optimized memset */
nop
L(update3return):
#ifdef USE_AS_STPNCPY
addi r3, r19, -1 /* update return value */
#endif
L(hop2return):
#ifndef USE_AS_STPNCPY
mr r3, r18 /* set return value */
#endif
addi r1, r1, FRAMESIZE /* restore stack pointer */
ld r0, 16(r1) /* read the saved link register */
ld r18, -16(r1) /* restore callers save register, r18 */
ld r19, -8(r1) /* restore callers save register, r19 */
mtlr r0 /* branch to link register */
blr /* return */
.p2align 4
L(update0):
mr r9, r19
.p2align 4
L(verifyByte):
rldicl. r8, r5, 0, 62
#ifdef USE_AS_STPNCPY
mr r3, r9
#endif
beq cr0, L(hop2return)
mtctr r8
addi r4, r4, -1
mr r19, r9
b L(oneBYone)
.p2align 4
L(proceed):
bdz L(done)
L(oneBYone):
lbzu r10, 1(r4) /* copy byte */
addi r19, r19, 1
addi r8, r8, -1
cmpdi cr7, r10, 0
stb r10, -1(r19)
bne cr7, L(proceed)
b L(zeroFill)
.p2align 4
L(done):
addi r1, r1, FRAMESIZE /* restore stack pointer */
#ifdef USE_AS_STPNCPY
mr r3, r19 /* set the return value */
#else
mr r3, r18 /* set the return value */
#endif
ld r0, 16(r1) /* read the saved link register */
ld r18, -16(r1) /* restore callers save register, r18 */
ld r19, -8(r1) /* restore callers save register, r19 */
mtlr r0 /* branch to link register */
blr /* return */
L(update1):
mr r0, r11
mr r19, r5
.p2align 4
L(leftDwords):
cmpdi cr7, r0, 0
mr r5, r19
bne cr7, L(dWordUnrollOFF)
b L(byte_by_byte)
.p2align 4
L(updtDestComputeN2ndByte):
addi r19, r19, 2 /* update dst by 2 */
subf r9, r19, r9 /* compute distance covered */
add r8, r9, r5
b L(zeroFill)
.p2align 4
L(updtDestComputeN3rdByte):
addi r19, r19, 3 /* update dst by 3 */
subf r9, r19, r9 /* compute distance covered */
add r8, r9, r5
b L(zeroFill)
.p2align 4
L(HopBy24):
addi r9, r9, 24 /* increment dst by 24 */
addi r4, r4, 24 /* increment src by 24 */
addi r5, r5, -24 /* decrement length 'n' by 24 */
addi r0, r11, -3 /* decrement loop counter */
b L(dWordUnrollOFF)
.p2align 4
L(update2):
mr r5, r19
b L(dWordUnrollOFF)
.p2align 4
L(HopBy40):
addi r9, r7, 40 /* increment dst by 40 */
addi r4, r6, 40 /* increment src by 40 */
addi r5, r5, -40 /* decrement length 'n' by 40 */
addi r0, r11, -5 /* decrement loop counter */
b L(dWordUnrollOFF)
L(update3):
mr r0, r11
b L(dWordUnrollOFF)
L(HopBy8):
addi r9, r3, 8 /* increment dst by 8 */
addi r4, r4, 8 /* increment src by 8 */
addi r5, r5, -8 /* decrement length 'n' by 8 */
addi r0, r11, -1 /* decrement loop counter */
b L(dWordUnrollOFF)
END(FUNC_NAME)
#ifndef USE_AS_STPNCPY
libc_hidden_builtin_def (strncpy)
#endif
|
