1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
|
/* Definition for thread-local data handling. nptl/i386 version.
Copyright (C) 2002-2012 Free Software Foundation, Inc.
This file is part of the GNU C Library.
The GNU C Library is free software; you can redistribute it and/or
modify it under the terms of the GNU Lesser General Public
License as published by the Free Software Foundation; either
version 2.1 of the License, or (at your option) any later version.
The GNU C Library is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
Lesser General Public License for more details.
You should have received a copy of the GNU Lesser General Public
License along with the GNU C Library; if not, see
<http://www.gnu.org/licenses/>. */
#ifndef _TLS_H
#define _TLS_H 1
#include <dl-sysdep.h>
#ifndef __ASSEMBLER__
# include <stdbool.h>
# include <stddef.h>
# include <stdint.h>
# include <stdlib.h>
# include <sysdep.h>
# include <libc-internal.h>
# include <kernel-features.h>
/* Type for the dtv. */
typedef union dtv
{
size_t counter;
struct
{
void *val;
bool is_static;
} pointer;
} dtv_t;
typedef struct
{
void *tcb; /* Pointer to the TCB. Not necessarily the
thread descriptor used by libpthread. */
dtv_t *dtv;
void *self; /* Pointer to the thread descriptor. */
int multiple_threads;
uintptr_t sysinfo;
uintptr_t stack_guard;
uintptr_t pointer_guard;
int gscope_flag;
#ifndef __ASSUME_PRIVATE_FUTEX
int private_futex;
#else
int __unused1;
#endif
/* Reservation of some values for the TM ABI. */
void *__private_tm[5];
} tcbhead_t;
# define TLS_MULTIPLE_THREADS_IN_TCB 1
#else /* __ASSEMBLER__ */
# include <tcb-offsets.h>
#endif
/* Alignment requirement for the stack. For IA-32 this is governed by
the SSE memory functions. */
#define STACK_ALIGN 16
#ifndef __ASSEMBLER__
/* Get system call information. */
# include <sysdep.h>
/* The old way: using LDT. */
/* Structure passed to `modify_ldt', 'set_thread_area', and 'clone' calls. */
struct user_desc
{
unsigned int entry_number;
unsigned long int base_addr;
unsigned int limit;
unsigned int seg_32bit:1;
unsigned int contents:2;
unsigned int read_exec_only:1;
unsigned int limit_in_pages:1;
unsigned int seg_not_present:1;
unsigned int useable:1;
unsigned int empty:25;
};
/* Initializing bit fields is slow. We speed it up by using a union. */
union user_desc_init
{
struct user_desc desc;
unsigned int vals[4];
};
/* Get the thread descriptor definition. */
# include <nptl/descr.h>
/* This is the size of the initial TCB. Can't be just sizeof (tcbhead_t),
because NPTL getpid, __libc_alloca_cutoff etc. need (almost) the whole
struct pthread even when not linked with -lpthread. */
# define TLS_INIT_TCB_SIZE sizeof (struct pthread)
/* Alignment requirements for the initial TCB. */
# define TLS_INIT_TCB_ALIGN __alignof__ (struct pthread)
/* This is the size of the TCB. */
# define TLS_TCB_SIZE sizeof (struct pthread)
/* Alignment requirements for the TCB. */
# define TLS_TCB_ALIGN __alignof__ (struct pthread)
/* The TCB can have any size and the memory following the address the
thread pointer points to is unspecified. Allocate the TCB there. */
# define TLS_TCB_AT_TP 1
/* Install the dtv pointer. The pointer passed is to the element with
index -1 which contain the length. */
# define INSTALL_DTV(descr, dtvp) \
((tcbhead_t *) (descr))->dtv = (dtvp) + 1
/* Install new dtv for current thread. */
# define INSTALL_NEW_DTV(dtvp) \
({ struct pthread *__pd; \
THREAD_SETMEM (__pd, header.dtv, (dtvp)); })
/* Return dtv of given thread descriptor. */
# define GET_DTV(descr) \
(((tcbhead_t *) (descr))->dtv)
#define THREAD_SELF_SYSINFO THREAD_GETMEM (THREAD_SELF, header.sysinfo)
#define THREAD_SYSINFO(pd) ((pd)->header.sysinfo)
/* Macros to load from and store into segment registers. */
# ifndef TLS_GET_GS
# define TLS_GET_GS() \
({ int __seg; __asm ("movw %%gs, %w0" : "=q" (__seg)); __seg & 0xffff; })
# endif
# ifndef TLS_SET_GS
# define TLS_SET_GS(val) \
__asm ("movw %w0, %%gs" :: "q" (val))
# endif
# ifndef __NR_set_thread_area
# define __NR_set_thread_area 243
# endif
# ifndef TLS_FLAG_WRITABLE
# define TLS_FLAG_WRITABLE 0x00000001
# endif
// XXX Enable for the real world.
#if 0
# ifndef __ASSUME_SET_THREAD_AREA
# error "we need set_thread_area"
# endif
#endif
# ifdef __PIC__
# define TLS_EBX_ARG "r"
# define TLS_LOAD_EBX "xchgl %3, %%ebx\n\t"
# else
# define TLS_EBX_ARG "b"
# define TLS_LOAD_EBX
# endif
#if defined NEED_DL_SYSINFO
# define INIT_SYSINFO \
_head->sysinfo = GLRO(dl_sysinfo)
#else
# define INIT_SYSINFO
#endif
#ifndef LOCK_PREFIX
# ifdef UP
# define LOCK_PREFIX /* nothing */
# else
# define LOCK_PREFIX "lock;"
# endif
#endif
/* Code to initially initialize the thread pointer. This might need
special attention since 'errno' is not yet available and if the
operation can cause a failure 'errno' must not be touched. */
# define TLS_INIT_TP(thrdescr, secondcall) \
({ void *_thrdescr = (thrdescr); \
tcbhead_t *_head = _thrdescr; \
union user_desc_init _segdescr; \
int _result; \
\
_head->tcb = _thrdescr; \
/* For now the thread descriptor is at the same address. */ \
_head->self = _thrdescr; \
/* New syscall handling support. */ \
INIT_SYSINFO; \
\
/* The 'entry_number' field. Let the kernel pick a value. */ \
if (secondcall) \
_segdescr.vals[0] = TLS_GET_GS () >> 3; \
else \
_segdescr.vals[0] = -1; \
/* The 'base_addr' field. Pointer to the TCB. */ \
_segdescr.vals[1] = (unsigned long int) _thrdescr; \
/* The 'limit' field. We use 4GB which is 0xfffff pages. */ \
_segdescr.vals[2] = 0xfffff; \
/* Collapsed value of the bitfield: \
.seg_32bit = 1 \
.contents = 0 \
.read_exec_only = 0 \
.limit_in_pages = 1 \
.seg_not_present = 0 \
.useable = 1 */ \
_segdescr.vals[3] = 0x51; \
\
/* Install the TLS. */ \
asm volatile (TLS_LOAD_EBX \
"int $0x80\n\t" \
TLS_LOAD_EBX \
: "=a" (_result), "=m" (_segdescr.desc.entry_number) \
: "0" (__NR_set_thread_area), \
TLS_EBX_ARG (&_segdescr.desc), "m" (_segdescr.desc)); \
\
if (_result == 0) \
/* We know the index in the GDT, now load the segment register. \
The use of the GDT is described by the value 3 in the lower \
three bits of the segment descriptor value. \
\
Note that we have to do this even if the numeric value of \
the descriptor does not change. Loading the segment register \
causes the segment information from the GDT to be loaded \
which is necessary since we have changed it. */ \
TLS_SET_GS (_segdescr.desc.entry_number * 8 + 3); \
\
_result == 0 ? NULL \
: "set_thread_area failed when setting up thread-local storage\n"; })
/* Return the address of the dtv for the current thread. */
# define THREAD_DTV() \
({ struct pthread *__pd; \
THREAD_GETMEM (__pd, header.dtv); })
/* Return the thread descriptor for the current thread.
The contained asm must *not* be marked volatile since otherwise
assignments like
pthread_descr self = thread_self();
do not get optimized away. */
# define THREAD_SELF \
({ struct pthread *__self; \
asm ("movl %%gs:%c1,%0" : "=r" (__self) \
: "i" (offsetof (struct pthread, header.self))); \
__self;})
/* Magic for libthread_db to know how to do THREAD_SELF. */
# define DB_THREAD_SELF \
REGISTER_THREAD_AREA (32, offsetof (struct user_regs_struct, xgs), 3) \
REGISTER_THREAD_AREA (64, 26 * 8, 3) /* x86-64's user_regs_struct->gs */
/* Read member of the thread descriptor directly. */
# define THREAD_GETMEM(descr, member) \
({ __typeof (descr->member) __value; \
if (sizeof (__value) == 1) \
asm volatile ("movb %%gs:%P2,%b0" \
: "=q" (__value) \
: "0" (0), "i" (offsetof (struct pthread, member))); \
else if (sizeof (__value) == 4) \
asm volatile ("movl %%gs:%P1,%0" \
: "=r" (__value) \
: "i" (offsetof (struct pthread, member))); \
else \
{ \
if (sizeof (__value) != 8) \
/* There should not be any value with a size other than 1, \
4 or 8. */ \
abort (); \
\
asm volatile ("movl %%gs:%P1,%%eax\n\t" \
"movl %%gs:%P2,%%edx" \
: "=A" (__value) \
: "i" (offsetof (struct pthread, member)), \
"i" (offsetof (struct pthread, member) + 4)); \
} \
__value; })
/* Same as THREAD_GETMEM, but the member offset can be non-constant. */
# define THREAD_GETMEM_NC(descr, member, idx) \
({ __typeof (descr->member[0]) __value; \
if (sizeof (__value) == 1) \
asm volatile ("movb %%gs:%P2(%3),%b0" \
: "=q" (__value) \
: "0" (0), "i" (offsetof (struct pthread, member[0])), \
"r" (idx)); \
else if (sizeof (__value) == 4) \
asm volatile ("movl %%gs:%P1(,%2,4),%0" \
: "=r" (__value) \
: "i" (offsetof (struct pthread, member[0])), \
"r" (idx)); \
else \
{ \
if (sizeof (__value) != 8) \
/* There should not be any value with a size other than 1, \
4 or 8. */ \
abort (); \
\
asm volatile ("movl %%gs:%P1(,%2,8),%%eax\n\t" \
"movl %%gs:4+%P1(,%2,8),%%edx" \
: "=&A" (__value) \
: "i" (offsetof (struct pthread, member[0])), \
"r" (idx)); \
} \
__value; })
/* Same as THREAD_SETMEM, but the member offset can be non-constant. */
# define THREAD_SETMEM(descr, member, value) \
({ if (sizeof (descr->member) == 1) \
asm volatile ("movb %b0,%%gs:%P1" : \
: "iq" (value), \
"i" (offsetof (struct pthread, member))); \
else if (sizeof (descr->member) == 4) \
asm volatile ("movl %0,%%gs:%P1" : \
: "ir" (value), \
"i" (offsetof (struct pthread, member))); \
else \
{ \
if (sizeof (descr->member) != 8) \
/* There should not be any value with a size other than 1, \
4 or 8. */ \
abort (); \
\
asm volatile ("movl %%eax,%%gs:%P1\n\t" \
"movl %%edx,%%gs:%P2" : \
: "A" ((uint64_t) cast_to_integer (value)), \
"i" (offsetof (struct pthread, member)), \
"i" (offsetof (struct pthread, member) + 4)); \
}})
/* Set member of the thread descriptor directly. */
# define THREAD_SETMEM_NC(descr, member, idx, value) \
({ if (sizeof (descr->member[0]) == 1) \
asm volatile ("movb %b0,%%gs:%P1(%2)" : \
: "iq" (value), \
"i" (offsetof (struct pthread, member)), \
"r" (idx)); \
else if (sizeof (descr->member[0]) == 4) \
asm volatile ("movl %0,%%gs:%P1(,%2,4)" : \
: "ir" (value), \
"i" (offsetof (struct pthread, member)), \
"r" (idx)); \
else \
{ \
if (sizeof (descr->member[0]) != 8) \
/* There should not be any value with a size other than 1, \
4 or 8. */ \
abort (); \
\
asm volatile ("movl %%eax,%%gs:%P1(,%2,8)\n\t" \
"movl %%edx,%%gs:4+%P1(,%2,8)" : \
: "A" ((uint64_t) cast_to_integer (value)), \
"i" (offsetof (struct pthread, member)), \
"r" (idx)); \
}})
/* Atomic compare and exchange on TLS, returning old value. */
#define THREAD_ATOMIC_CMPXCHG_VAL(descr, member, newval, oldval) \
({ __typeof (descr->member) __ret; \
__typeof (oldval) __old = (oldval); \
if (sizeof (descr->member) == 4) \
asm volatile (LOCK_PREFIX "cmpxchgl %2, %%gs:%P3" \
: "=a" (__ret) \
: "0" (__old), "r" (newval), \
"i" (offsetof (struct pthread, member))); \
else \
/* Not necessary for other sizes in the moment. */ \
abort (); \
__ret; })
/* Atomic logical and. */
#define THREAD_ATOMIC_AND(descr, member, val) \
(void) ({ if (sizeof ((descr)->member) == 4) \
asm volatile (LOCK_PREFIX "andl %1, %%gs:%P0" \
:: "i" (offsetof (struct pthread, member)), \
"ir" (val)); \
else \
/* Not necessary for other sizes in the moment. */ \
abort (); })
/* Atomic set bit. */
#define THREAD_ATOMIC_BIT_SET(descr, member, bit) \
(void) ({ if (sizeof ((descr)->member) == 4) \
asm volatile (LOCK_PREFIX "orl %1, %%gs:%P0" \
:: "i" (offsetof (struct pthread, member)), \
"ir" (1 << (bit))); \
else \
/* Not necessary for other sizes in the moment. */ \
abort (); })
/* Call the user-provided thread function. */
#define CALL_THREAD_FCT(descr) \
({ void *__res; \
int __ignore1, __ignore2; \
asm volatile ("pushl %%eax\n\t" \
"pushl %%eax\n\t" \
"pushl %%eax\n\t" \
"pushl %%gs:%P4\n\t" \
"call *%%gs:%P3\n\t" \
"addl $16, %%esp" \
: "=a" (__res), "=c" (__ignore1), "=d" (__ignore2) \
: "i" (offsetof (struct pthread, start_routine)), \
"i" (offsetof (struct pthread, arg))); \
__res; })
/* Set the stack guard field in TCB head. */
#define THREAD_SET_STACK_GUARD(value) \
THREAD_SETMEM (THREAD_SELF, header.stack_guard, value)
#define THREAD_COPY_STACK_GUARD(descr) \
((descr)->header.stack_guard \
= THREAD_GETMEM (THREAD_SELF, header.stack_guard))
/* Set the pointer guard field in the TCB head. */
#define THREAD_SET_POINTER_GUARD(value) \
THREAD_SETMEM (THREAD_SELF, header.pointer_guard, value)
#define THREAD_COPY_POINTER_GUARD(descr) \
((descr)->header.pointer_guard \
= THREAD_GETMEM (THREAD_SELF, header.pointer_guard))
/* Get and set the global scope generation counter in the TCB head. */
#define THREAD_GSCOPE_FLAG_UNUSED 0
#define THREAD_GSCOPE_FLAG_USED 1
#define THREAD_GSCOPE_FLAG_WAIT 2
#define THREAD_GSCOPE_RESET_FLAG() \
do \
{ int __res; \
asm volatile ("xchgl %0, %%gs:%P1" \
: "=r" (__res) \
: "i" (offsetof (struct pthread, header.gscope_flag)), \
"0" (THREAD_GSCOPE_FLAG_UNUSED)); \
if (__res == THREAD_GSCOPE_FLAG_WAIT) \
lll_futex_wake (&THREAD_SELF->header.gscope_flag, 1, LLL_PRIVATE); \
} \
while (0)
#define THREAD_GSCOPE_SET_FLAG() \
THREAD_SETMEM (THREAD_SELF, header.gscope_flag, THREAD_GSCOPE_FLAG_USED)
#define THREAD_GSCOPE_WAIT() \
GL(dl_wait_lookup_done) ()
#endif /* __ASSEMBLER__ */
#endif /* tls.h */
|
