Loading include/asm-x86/uaccess_64.h +227 −149 Original line number Diff line number Diff line Loading @@ -31,19 +31,23 @@ #define segment_eq(a, b) ((a).seg == (b).seg) #define __addr_ok(addr) (!((unsigned long)(addr) & (current_thread_info()->addr_limit.seg))) #define __addr_ok(addr) (!((unsigned long)(addr) & \ (current_thread_info()->addr_limit.seg))) /* * Uhhuh, this needs 65-bit arithmetic. We have a carry.. */ #define __range_not_ok(addr,size) ({ \ #define __range_not_ok(addr, size) \ ({ \ unsigned long flag, roksum; \ __chk_user_ptr(addr); \ asm("# range_ok\n\r" \ "addq %3,%1 ; sbbq %0,%0 ; cmpq %1,%4 ; sbbq $0,%0" \ : "=&r" (flag), "=r" (roksum) \ :"1" (addr),"g" ((long)(size)),"g" (current_thread_info()->addr_limit.seg)); \ flag; }) : "1" (addr), "g" ((long)(size)), \ "g" (current_thread_info()->addr_limit.seg)); \ flag; \ }) #define access_ok(type, addr, size) (__range_not_ok(addr, size) == 0) Loading @@ -60,8 +64,7 @@ * on our cache or tlb entries. */ struct exception_table_entry { struct exception_table_entry { unsigned long insn, fixup; }; Loading Loading @@ -90,17 +93,30 @@ extern int fixup_exception(struct pt_regs *regs); : "c" (ptr) \ : "r8") /* Careful: we have to cast the result to the type of the pointer for sign reasons */ /* Careful: we have to cast the result to the type of the pointer * for sign reasons */ #define get_user(x, ptr) \ ({ unsigned long __val_gu; \ ({ \ unsigned long __val_gu; \ int __ret_gu; \ __chk_user_ptr(ptr); \ switch (sizeof(*(ptr))) { \ case 1: __get_user_x(1,__ret_gu,__val_gu,ptr); break; \ case 2: __get_user_x(2,__ret_gu,__val_gu,ptr); break; \ case 4: __get_user_x(4,__ret_gu,__val_gu,ptr); break; \ case 8: __get_user_x(8,__ret_gu,__val_gu,ptr); break; \ default: __get_user_bad(); break; \ case 1: \ __get_user_x(1, __ret_gu, __val_gu, ptr); \ break; \ case 2: \ __get_user_x(2, __ret_gu, __val_gu, ptr); \ break; \ case 4: \ __get_user_x(4, __ret_gu, __val_gu, ptr); \ break; \ case 8: \ __get_user_x(8, __ret_gu, __val_gu, ptr); \ break; \ default: \ __get_user_bad(); \ break; \ } \ (x) = (__force typeof(*(ptr)))__val_gu; \ __ret_gu; \ Loading Loading @@ -142,11 +158,20 @@ extern void __put_user_bad(void); int __pu_err; \ typeof(*(ptr)) __user *__pu_addr = (ptr); \ switch (size) { \ case 1: __put_user_x(1,__pu_err,x,__pu_addr); break; \ case 2: __put_user_x(2,__pu_err,x,__pu_addr); break; \ case 4: __put_user_x(4,__pu_err,x,__pu_addr); break; \ case 8: __put_user_x(8,__pu_err,x,__pu_addr); break; \ default: __put_user_bad(); \ case 1: \ __put_user_x(1, __pu_err, x, __pu_addr); \ break; \ case 2: \ __put_user_x(2, __pu_err, x, __pu_addr); \ break; \ case 4: \ __put_user_x(4, __pu_err, x, __pu_addr); \ break; \ case 8: \ __put_user_x(8, __pu_err, x, __pu_addr); \ break; \ default: \ __put_user_bad(); \ } \ __pu_err; \ }) Loading @@ -156,11 +181,20 @@ do { \ retval = 0; \ __chk_user_ptr(ptr); \ switch (size) { \ case 1: __put_user_asm(x,ptr,retval,"b","b","iq",-EFAULT); break;\ case 2: __put_user_asm(x,ptr,retval,"w","w","ir",-EFAULT); break;\ case 4: __put_user_asm(x,ptr,retval,"l","k","ir",-EFAULT); break;\ case 8: __put_user_asm(x,ptr,retval,"q","","Zr",-EFAULT); break;\ default: __put_user_bad(); \ case 1: \ __put_user_asm(x, ptr, retval, "b", "b", "iq", -EFAULT);\ break; \ case 2: \ __put_user_asm(x, ptr, retval, "w", "w", "ir", -EFAULT);\ break; \ case 4: \ __put_user_asm(x, ptr, retval, "l", "k", "ir", -EFAULT);\ break; \ case 8: \ __put_user_asm(x, ptr, retval, "q", "", "Zr", -EFAULT); \ break; \ default: \ __put_user_bad(); \ } \ } while (0) Loading @@ -174,8 +208,7 @@ struct __large_struct { unsigned long buf[100]; }; * aliasing issues. */ #define __put_user_asm(x, addr, err, itype, rtype, ltype, errno) \ asm volatile( \ "1: mov"itype" %"rtype"1,%2\n" \ asm volatile("1: mov"itype" %"rtype"1,%2\n" \ "2:\n" \ ".section .fixup, \"ax\"\n" \ "3: mov %3,%0\n" \ Loading Loading @@ -206,17 +239,25 @@ do { \ retval = 0; \ __chk_user_ptr(ptr); \ switch (size) { \ case 1: __get_user_asm(x,ptr,retval,"b","b","=q",-EFAULT); break;\ case 2: __get_user_asm(x,ptr,retval,"w","w","=r",-EFAULT); break;\ case 4: __get_user_asm(x,ptr,retval,"l","k","=r",-EFAULT); break;\ case 8: __get_user_asm(x,ptr,retval,"q","","=r",-EFAULT); break;\ default: (x) = __get_user_bad(); \ case 1: \ __get_user_asm(x, ptr, retval, "b", "b", "=q", -EFAULT);\ break; \ case 2: \ __get_user_asm(x, ptr, retval, "w", "w", "=r", -EFAULT);\ break; \ case 4: \ __get_user_asm(x, ptr, retval, "l", "k", "=r", -EFAULT);\ break; \ case 8: \ __get_user_asm(x, ptr, retval, "q", "", "=r", -EFAULT); \ break; \ default: \ (x) = __get_user_bad(); \ } \ } while (0) #define __get_user_asm(x, addr, err, itype, rtype, ltype, errno) \ asm volatile( \ "1: mov"itype" %2,%"rtype"1\n" \ asm volatile("1: mov"itype" %2,%"rtype"1\n" \ "2:\n" \ ".section .fixup, \"ax\"\n" \ "3: mov %3,%0\n" \ Loading Loading @@ -249,23 +290,35 @@ int __copy_from_user(void *dst, const void __user *src, unsigned size) if (!__builtin_constant_p(size)) return copy_user_generic(dst, (__force void *)src, size); switch (size) { case 1:__get_user_asm(*(u8*)dst,(u8 __user *)src,ret,"b","b","=q",1); case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src, ret, "b", "b", "=q", 1); return ret; case 2:__get_user_asm(*(u16*)dst,(u16 __user *)src,ret,"w","w","=r",2); case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src, ret, "w", "w", "=r", 2); return ret; case 4:__get_user_asm(*(u32*)dst,(u32 __user *)src,ret,"l","k","=r",4); case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src, ret, "l", "k", "=r", 4); return ret; case 8:__get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",8); case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src, ret, "q", "", "=r", 8); return ret; case 10: __get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16); if (unlikely(ret)) return ret; __get_user_asm(*(u16*)(8+(char*)dst),(u16 __user *)(8+(char __user *)src),ret,"w","w","=r",2); __get_user_asm(*(u64 *)dst, (u64 __user *)src, ret, "q", "", "=r", 16); if (unlikely(ret)) return ret; __get_user_asm(*(u16 *)(8 + (char *)dst), (u16 __user *)(8 + (char __user *)src), ret, "w", "w", "=r", 2); return ret; case 16: __get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16); if (unlikely(ret)) return ret; __get_user_asm(*(u64*)(8+(char*)dst),(u64 __user *)(8+(char __user *)src),ret,"q","","=r",8); __get_user_asm(*(u64 *)dst, (u64 __user *)src, ret, "q", "", "=r", 16); if (unlikely(ret)) return ret; __get_user_asm(*(u64 *)(8 + (char *)dst), (u64 __user *)(8 + (char __user *)src), ret, "q", "", "=r", 8); return ret; default: return copy_user_generic(dst, (__force void *)src, size); Loading @@ -279,25 +332,35 @@ int __copy_to_user(void __user *dst, const void *src, unsigned size) if (!__builtin_constant_p(size)) return copy_user_generic((__force void *)dst, src, size); switch (size) { case 1:__put_user_asm(*(u8*)src,(u8 __user *)dst,ret,"b","b","iq",1); case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst, ret, "b", "b", "iq", 1); return ret; case 2:__put_user_asm(*(u16*)src,(u16 __user *)dst,ret,"w","w","ir",2); case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst, ret, "w", "w", "ir", 2); return ret; case 4:__put_user_asm(*(u32*)src,(u32 __user *)dst,ret,"l","k","ir",4); case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst, ret, "l", "k", "ir", 4); return ret; case 8:__put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",8); case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst, ret, "q", "", "ir", 8); return ret; case 10: __put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",10); if (unlikely(ret)) return ret; __put_user_asm(*(u64 *)src, (u64 __user *)dst, ret, "q", "", "ir", 10); if (unlikely(ret)) return ret; asm("":::"memory"); __put_user_asm(4[(u16*)src],4+(u16 __user *)dst,ret,"w","w","ir",2); __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst, ret, "w", "w", "ir", 2); return ret; case 16: __put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",16); if (unlikely(ret)) return ret; __put_user_asm(*(u64 *)src, (u64 __user *)dst, ret, "q", "", "ir", 16); if (unlikely(ret)) return ret; asm("":::"memory"); __put_user_asm(1[(u64*)src],1+(u64 __user *)dst,ret,"q","","ir",8); __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst, ret, "q", "", "ir", 8); return ret; default: return copy_user_generic((__force void *)dst, src, size); Loading @@ -309,39 +372,49 @@ int __copy_in_user(void __user *dst, const void __user *src, unsigned size) { int ret = 0; if (!__builtin_constant_p(size)) return copy_user_generic((__force void *)dst,(__force void *)src,size); return copy_user_generic((__force void *)dst, (__force void *)src, size); switch (size) { case 1: { u8 tmp; __get_user_asm(tmp,(u8 __user *)src,ret,"b","b","=q",1); __get_user_asm(tmp, (u8 __user *)src, ret, "b", "b", "=q", 1); if (likely(!ret)) __put_user_asm(tmp,(u8 __user *)dst,ret,"b","b","iq",1); __put_user_asm(tmp, (u8 __user *)dst, ret, "b", "b", "iq", 1); return ret; } case 2: { u16 tmp; __get_user_asm(tmp,(u16 __user *)src,ret,"w","w","=r",2); __get_user_asm(tmp, (u16 __user *)src, ret, "w", "w", "=r", 2); if (likely(!ret)) __put_user_asm(tmp,(u16 __user *)dst,ret,"w","w","ir",2); __put_user_asm(tmp, (u16 __user *)dst, ret, "w", "w", "ir", 2); return ret; } case 4: { u32 tmp; __get_user_asm(tmp,(u32 __user *)src,ret,"l","k","=r",4); __get_user_asm(tmp, (u32 __user *)src, ret, "l", "k", "=r", 4); if (likely(!ret)) __put_user_asm(tmp,(u32 __user *)dst,ret,"l","k","ir",4); __put_user_asm(tmp, (u32 __user *)dst, ret, "l", "k", "ir", 4); return ret; } case 8: { u64 tmp; __get_user_asm(tmp,(u64 __user *)src,ret,"q","","=r",8); __get_user_asm(tmp, (u64 __user *)src, ret, "q", "", "=r", 8); if (likely(!ret)) __put_user_asm(tmp,(u64 __user *)dst,ret,"q","","ir",8); __put_user_asm(tmp, (u64 __user *)dst, ret, "q", "", "ir", 8); return ret; } default: return copy_user_generic((__force void *)dst,(__force void *)src,size); return copy_user_generic((__force void *)dst, (__force void *)src, size); } } Loading @@ -355,7 +428,8 @@ __must_check long strlen_user(const char __user *str); __must_check unsigned long clear_user(void __user *mem, unsigned long len); __must_check unsigned long __clear_user(void __user *mem, unsigned long len); __must_check long __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size); __must_check long __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size); static __must_check __always_inline int __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size) Loading @@ -364,15 +438,19 @@ __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size) } #define ARCH_HAS_NOCACHE_UACCESS 1 extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size, int zerorest); extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size, int zerorest); static inline int __copy_from_user_nocache(void *dst, const void __user *src, unsigned size) static inline int __copy_from_user_nocache(void *dst, const void __user *src, unsigned size) { might_sleep(); return __copy_user_nocache(dst, src, size, 1); } static inline int __copy_from_user_inatomic_nocache(void *dst, const void __user *src, unsigned size) static inline int __copy_from_user_inatomic_nocache(void *dst, const void __user *src, unsigned size) { return __copy_user_nocache(dst, src, size, 0); } Loading Loading
include/asm-x86/uaccess_64.h +227 −149 Original line number Diff line number Diff line Loading @@ -31,19 +31,23 @@ #define segment_eq(a, b) ((a).seg == (b).seg) #define __addr_ok(addr) (!((unsigned long)(addr) & (current_thread_info()->addr_limit.seg))) #define __addr_ok(addr) (!((unsigned long)(addr) & \ (current_thread_info()->addr_limit.seg))) /* * Uhhuh, this needs 65-bit arithmetic. We have a carry.. */ #define __range_not_ok(addr,size) ({ \ #define __range_not_ok(addr, size) \ ({ \ unsigned long flag, roksum; \ __chk_user_ptr(addr); \ asm("# range_ok\n\r" \ "addq %3,%1 ; sbbq %0,%0 ; cmpq %1,%4 ; sbbq $0,%0" \ : "=&r" (flag), "=r" (roksum) \ :"1" (addr),"g" ((long)(size)),"g" (current_thread_info()->addr_limit.seg)); \ flag; }) : "1" (addr), "g" ((long)(size)), \ "g" (current_thread_info()->addr_limit.seg)); \ flag; \ }) #define access_ok(type, addr, size) (__range_not_ok(addr, size) == 0) Loading @@ -60,8 +64,7 @@ * on our cache or tlb entries. */ struct exception_table_entry { struct exception_table_entry { unsigned long insn, fixup; }; Loading Loading @@ -90,17 +93,30 @@ extern int fixup_exception(struct pt_regs *regs); : "c" (ptr) \ : "r8") /* Careful: we have to cast the result to the type of the pointer for sign reasons */ /* Careful: we have to cast the result to the type of the pointer * for sign reasons */ #define get_user(x, ptr) \ ({ unsigned long __val_gu; \ ({ \ unsigned long __val_gu; \ int __ret_gu; \ __chk_user_ptr(ptr); \ switch (sizeof(*(ptr))) { \ case 1: __get_user_x(1,__ret_gu,__val_gu,ptr); break; \ case 2: __get_user_x(2,__ret_gu,__val_gu,ptr); break; \ case 4: __get_user_x(4,__ret_gu,__val_gu,ptr); break; \ case 8: __get_user_x(8,__ret_gu,__val_gu,ptr); break; \ default: __get_user_bad(); break; \ case 1: \ __get_user_x(1, __ret_gu, __val_gu, ptr); \ break; \ case 2: \ __get_user_x(2, __ret_gu, __val_gu, ptr); \ break; \ case 4: \ __get_user_x(4, __ret_gu, __val_gu, ptr); \ break; \ case 8: \ __get_user_x(8, __ret_gu, __val_gu, ptr); \ break; \ default: \ __get_user_bad(); \ break; \ } \ (x) = (__force typeof(*(ptr)))__val_gu; \ __ret_gu; \ Loading Loading @@ -142,11 +158,20 @@ extern void __put_user_bad(void); int __pu_err; \ typeof(*(ptr)) __user *__pu_addr = (ptr); \ switch (size) { \ case 1: __put_user_x(1,__pu_err,x,__pu_addr); break; \ case 2: __put_user_x(2,__pu_err,x,__pu_addr); break; \ case 4: __put_user_x(4,__pu_err,x,__pu_addr); break; \ case 8: __put_user_x(8,__pu_err,x,__pu_addr); break; \ default: __put_user_bad(); \ case 1: \ __put_user_x(1, __pu_err, x, __pu_addr); \ break; \ case 2: \ __put_user_x(2, __pu_err, x, __pu_addr); \ break; \ case 4: \ __put_user_x(4, __pu_err, x, __pu_addr); \ break; \ case 8: \ __put_user_x(8, __pu_err, x, __pu_addr); \ break; \ default: \ __put_user_bad(); \ } \ __pu_err; \ }) Loading @@ -156,11 +181,20 @@ do { \ retval = 0; \ __chk_user_ptr(ptr); \ switch (size) { \ case 1: __put_user_asm(x,ptr,retval,"b","b","iq",-EFAULT); break;\ case 2: __put_user_asm(x,ptr,retval,"w","w","ir",-EFAULT); break;\ case 4: __put_user_asm(x,ptr,retval,"l","k","ir",-EFAULT); break;\ case 8: __put_user_asm(x,ptr,retval,"q","","Zr",-EFAULT); break;\ default: __put_user_bad(); \ case 1: \ __put_user_asm(x, ptr, retval, "b", "b", "iq", -EFAULT);\ break; \ case 2: \ __put_user_asm(x, ptr, retval, "w", "w", "ir", -EFAULT);\ break; \ case 4: \ __put_user_asm(x, ptr, retval, "l", "k", "ir", -EFAULT);\ break; \ case 8: \ __put_user_asm(x, ptr, retval, "q", "", "Zr", -EFAULT); \ break; \ default: \ __put_user_bad(); \ } \ } while (0) Loading @@ -174,8 +208,7 @@ struct __large_struct { unsigned long buf[100]; }; * aliasing issues. */ #define __put_user_asm(x, addr, err, itype, rtype, ltype, errno) \ asm volatile( \ "1: mov"itype" %"rtype"1,%2\n" \ asm volatile("1: mov"itype" %"rtype"1,%2\n" \ "2:\n" \ ".section .fixup, \"ax\"\n" \ "3: mov %3,%0\n" \ Loading Loading @@ -206,17 +239,25 @@ do { \ retval = 0; \ __chk_user_ptr(ptr); \ switch (size) { \ case 1: __get_user_asm(x,ptr,retval,"b","b","=q",-EFAULT); break;\ case 2: __get_user_asm(x,ptr,retval,"w","w","=r",-EFAULT); break;\ case 4: __get_user_asm(x,ptr,retval,"l","k","=r",-EFAULT); break;\ case 8: __get_user_asm(x,ptr,retval,"q","","=r",-EFAULT); break;\ default: (x) = __get_user_bad(); \ case 1: \ __get_user_asm(x, ptr, retval, "b", "b", "=q", -EFAULT);\ break; \ case 2: \ __get_user_asm(x, ptr, retval, "w", "w", "=r", -EFAULT);\ break; \ case 4: \ __get_user_asm(x, ptr, retval, "l", "k", "=r", -EFAULT);\ break; \ case 8: \ __get_user_asm(x, ptr, retval, "q", "", "=r", -EFAULT); \ break; \ default: \ (x) = __get_user_bad(); \ } \ } while (0) #define __get_user_asm(x, addr, err, itype, rtype, ltype, errno) \ asm volatile( \ "1: mov"itype" %2,%"rtype"1\n" \ asm volatile("1: mov"itype" %2,%"rtype"1\n" \ "2:\n" \ ".section .fixup, \"ax\"\n" \ "3: mov %3,%0\n" \ Loading Loading @@ -249,23 +290,35 @@ int __copy_from_user(void *dst, const void __user *src, unsigned size) if (!__builtin_constant_p(size)) return copy_user_generic(dst, (__force void *)src, size); switch (size) { case 1:__get_user_asm(*(u8*)dst,(u8 __user *)src,ret,"b","b","=q",1); case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src, ret, "b", "b", "=q", 1); return ret; case 2:__get_user_asm(*(u16*)dst,(u16 __user *)src,ret,"w","w","=r",2); case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src, ret, "w", "w", "=r", 2); return ret; case 4:__get_user_asm(*(u32*)dst,(u32 __user *)src,ret,"l","k","=r",4); case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src, ret, "l", "k", "=r", 4); return ret; case 8:__get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",8); case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src, ret, "q", "", "=r", 8); return ret; case 10: __get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16); if (unlikely(ret)) return ret; __get_user_asm(*(u16*)(8+(char*)dst),(u16 __user *)(8+(char __user *)src),ret,"w","w","=r",2); __get_user_asm(*(u64 *)dst, (u64 __user *)src, ret, "q", "", "=r", 16); if (unlikely(ret)) return ret; __get_user_asm(*(u16 *)(8 + (char *)dst), (u16 __user *)(8 + (char __user *)src), ret, "w", "w", "=r", 2); return ret; case 16: __get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16); if (unlikely(ret)) return ret; __get_user_asm(*(u64*)(8+(char*)dst),(u64 __user *)(8+(char __user *)src),ret,"q","","=r",8); __get_user_asm(*(u64 *)dst, (u64 __user *)src, ret, "q", "", "=r", 16); if (unlikely(ret)) return ret; __get_user_asm(*(u64 *)(8 + (char *)dst), (u64 __user *)(8 + (char __user *)src), ret, "q", "", "=r", 8); return ret; default: return copy_user_generic(dst, (__force void *)src, size); Loading @@ -279,25 +332,35 @@ int __copy_to_user(void __user *dst, const void *src, unsigned size) if (!__builtin_constant_p(size)) return copy_user_generic((__force void *)dst, src, size); switch (size) { case 1:__put_user_asm(*(u8*)src,(u8 __user *)dst,ret,"b","b","iq",1); case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst, ret, "b", "b", "iq", 1); return ret; case 2:__put_user_asm(*(u16*)src,(u16 __user *)dst,ret,"w","w","ir",2); case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst, ret, "w", "w", "ir", 2); return ret; case 4:__put_user_asm(*(u32*)src,(u32 __user *)dst,ret,"l","k","ir",4); case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst, ret, "l", "k", "ir", 4); return ret; case 8:__put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",8); case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst, ret, "q", "", "ir", 8); return ret; case 10: __put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",10); if (unlikely(ret)) return ret; __put_user_asm(*(u64 *)src, (u64 __user *)dst, ret, "q", "", "ir", 10); if (unlikely(ret)) return ret; asm("":::"memory"); __put_user_asm(4[(u16*)src],4+(u16 __user *)dst,ret,"w","w","ir",2); __put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst, ret, "w", "w", "ir", 2); return ret; case 16: __put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",16); if (unlikely(ret)) return ret; __put_user_asm(*(u64 *)src, (u64 __user *)dst, ret, "q", "", "ir", 16); if (unlikely(ret)) return ret; asm("":::"memory"); __put_user_asm(1[(u64*)src],1+(u64 __user *)dst,ret,"q","","ir",8); __put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst, ret, "q", "", "ir", 8); return ret; default: return copy_user_generic((__force void *)dst, src, size); Loading @@ -309,39 +372,49 @@ int __copy_in_user(void __user *dst, const void __user *src, unsigned size) { int ret = 0; if (!__builtin_constant_p(size)) return copy_user_generic((__force void *)dst,(__force void *)src,size); return copy_user_generic((__force void *)dst, (__force void *)src, size); switch (size) { case 1: { u8 tmp; __get_user_asm(tmp,(u8 __user *)src,ret,"b","b","=q",1); __get_user_asm(tmp, (u8 __user *)src, ret, "b", "b", "=q", 1); if (likely(!ret)) __put_user_asm(tmp,(u8 __user *)dst,ret,"b","b","iq",1); __put_user_asm(tmp, (u8 __user *)dst, ret, "b", "b", "iq", 1); return ret; } case 2: { u16 tmp; __get_user_asm(tmp,(u16 __user *)src,ret,"w","w","=r",2); __get_user_asm(tmp, (u16 __user *)src, ret, "w", "w", "=r", 2); if (likely(!ret)) __put_user_asm(tmp,(u16 __user *)dst,ret,"w","w","ir",2); __put_user_asm(tmp, (u16 __user *)dst, ret, "w", "w", "ir", 2); return ret; } case 4: { u32 tmp; __get_user_asm(tmp,(u32 __user *)src,ret,"l","k","=r",4); __get_user_asm(tmp, (u32 __user *)src, ret, "l", "k", "=r", 4); if (likely(!ret)) __put_user_asm(tmp,(u32 __user *)dst,ret,"l","k","ir",4); __put_user_asm(tmp, (u32 __user *)dst, ret, "l", "k", "ir", 4); return ret; } case 8: { u64 tmp; __get_user_asm(tmp,(u64 __user *)src,ret,"q","","=r",8); __get_user_asm(tmp, (u64 __user *)src, ret, "q", "", "=r", 8); if (likely(!ret)) __put_user_asm(tmp,(u64 __user *)dst,ret,"q","","ir",8); __put_user_asm(tmp, (u64 __user *)dst, ret, "q", "", "ir", 8); return ret; } default: return copy_user_generic((__force void *)dst,(__force void *)src,size); return copy_user_generic((__force void *)dst, (__force void *)src, size); } } Loading @@ -355,7 +428,8 @@ __must_check long strlen_user(const char __user *str); __must_check unsigned long clear_user(void __user *mem, unsigned long len); __must_check unsigned long __clear_user(void __user *mem, unsigned long len); __must_check long __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size); __must_check long __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size); static __must_check __always_inline int __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size) Loading @@ -364,15 +438,19 @@ __copy_to_user_inatomic(void __user *dst, const void *src, unsigned size) } #define ARCH_HAS_NOCACHE_UACCESS 1 extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size, int zerorest); extern long __copy_user_nocache(void *dst, const void __user *src, unsigned size, int zerorest); static inline int __copy_from_user_nocache(void *dst, const void __user *src, unsigned size) static inline int __copy_from_user_nocache(void *dst, const void __user *src, unsigned size) { might_sleep(); return __copy_user_nocache(dst, src, size, 1); } static inline int __copy_from_user_inatomic_nocache(void *dst, const void __user *src, unsigned size) static inline int __copy_from_user_inatomic_nocache(void *dst, const void __user *src, unsigned size) { return __copy_user_nocache(dst, src, size, 0); } Loading