// Copyright 2016 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. // The gccgo version of mem_*.go. package runtime import ( "unsafe" ) // Functions called by C code. //go:linkname sysAlloc //go:linkname sysFree //extern __go_mmap func sysMmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uintptr) unsafe.Pointer //extern munmap func munmap(addr unsafe.Pointer, length uintptr) int32 //extern madvise func madvise(addr unsafe.Pointer, n uintptr, flags int32) int32 var mmapFD = int32(-1) var devZero = []byte("/dev/zero\x00") func init() { if _MAP_ANON == 0 { mmapFD = open(&devZero[0], 0 /* O_RDONLY */, 0) if mmapFD < 0 { println("open /dev/zero: errno=", errno()) exit(2) } } } func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uintptr) (unsafe.Pointer, int) { p := sysMmap(addr, n, prot, flags, fd, off) if uintptr(p) == _MAP_FAILED { return nil, errno() } return p, 0 } // Don't split the stack as this method may be invoked without a valid G, which // prevents us from allocating more stack. // //go:nosplit func sysAlloc(n uintptr, sysStat *sysMemStat) unsafe.Pointer { p, err := mmap(nil, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_PRIVATE, mmapFD, 0) if err != 0 { if err == _EACCES { print("runtime: mmap: access denied\n") exit(2) } if err == _EAGAIN { print("runtime: mmap: too much locked memory (check 'ulimit -l').\n") exit(2) } return nil } sysStat.add(int64(n)) return p } func sysUnused(v unsafe.Pointer, n uintptr) { // By default, Linux's "transparent huge page" support will // merge pages into a huge page if there's even a single // present regular page, undoing the effects of the DONTNEED // below. On amd64, that means khugepaged can turn a single // 4KB page to 2MB, bloating the process's RSS by as much as // 512X. (See issue #8832 and Linux kernel bug // https://bugzilla.kernel.org/show_bug.cgi?id=93111) // // To work around this, we explicitly disable transparent huge // pages when we release pages of the heap. However, we have // to do this carefully because changing this flag tends to // split the VMA (memory mapping) containing v in to three // VMAs in order to track the different values of the // MADV_NOHUGEPAGE flag in the different regions. There's a // default limit of 65530 VMAs per address space (sysctl // vm.max_map_count), so we must be careful not to create too // many VMAs (see issue #12233). // // Since huge pages are huge, there's little use in adjusting // the MADV_NOHUGEPAGE flag on a fine granularity, so we avoid // exploding the number of VMAs by only adjusting the // MADV_NOHUGEPAGE flag on a large granularity. This still // gets most of the benefit of huge pages while keeping the // number of VMAs under control. With hugePageSize = 2MB, even // a pessimal heap can reach 128GB before running out of VMAs. if physHugePageSize != 0 && _MADV_NOHUGEPAGE != 0 { // If it's a large allocation, we want to leave huge // pages enabled. Hence, we only adjust the huge page // flag on the huge pages containing v and v+n-1, and // only if those aren't aligned. var head, tail uintptr if uintptr(v)%physHugePageSize != 0 { // Compute huge page containing v. head = uintptr(v) &^ (physHugePageSize - 1) } if (uintptr(v)+n)%physHugePageSize != 0 { // Compute huge page containing v+n-1. tail = (uintptr(v) + n - 1) &^ (physHugePageSize - 1) } // Note that madvise will return EINVAL if the flag is // already set, which is quite likely. We ignore // errors. if head != 0 && head+physHugePageSize == tail { // head and tail are different but adjacent, // so do this in one call. madvise(unsafe.Pointer(head), 2*physHugePageSize, _MADV_NOHUGEPAGE) } else { // Advise the huge pages containing v and v+n-1. if head != 0 { madvise(unsafe.Pointer(head), physHugePageSize, _MADV_NOHUGEPAGE) } if tail != 0 && tail != head { madvise(unsafe.Pointer(tail), physHugePageSize, _MADV_NOHUGEPAGE) } } } if uintptr(v)&(physPageSize-1) != 0 || n&(physPageSize-1) != 0 { // madvise will round this to any physical page // *covered* by this range, so an unaligned madvise // will release more memory than intended. throw("unaligned sysUnused") } if _MADV_DONTNEED != 0 { madvise(v, n, _MADV_DONTNEED) } else if _MADV_FREE != 0 { madvise(v, n, _MADV_FREE) } } func sysUsed(v unsafe.Pointer, n uintptr) { // Partially undo the NOHUGEPAGE marks from sysUnused // for whole huge pages between v and v+n. This may // leave huge pages off at the end points v and v+n // even though allocations may cover these entire huge // pages. We could detect this and undo NOHUGEPAGE on // the end points as well, but it's probably not worth // the cost because when neighboring allocations are // freed sysUnused will just set NOHUGEPAGE again. sysHugePage(v, n) } func sysHugePage(v unsafe.Pointer, n uintptr) { if physHugePageSize != 0 && _MADV_HUGEPAGE != 0 { // Round v up to a huge page boundary. beg := (uintptr(v) + (physHugePageSize - 1)) &^ (physHugePageSize - 1) // Round v+n down to a huge page boundary. end := (uintptr(v) + n) &^ (physHugePageSize - 1) if beg < end { madvise(unsafe.Pointer(beg), end-beg, _MADV_HUGEPAGE) } } } // Don't split the stack as this function may be invoked without a valid G, // which prevents us from allocating more stack. // //go:nosplit func sysFree(v unsafe.Pointer, n uintptr, sysStat *sysMemStat) { sysStat.add(-int64(n)) munmap(v, n) } func sysFault(v unsafe.Pointer, n uintptr) { mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE|_MAP_FIXED, mmapFD, 0) } func sysReserve(v unsafe.Pointer, n uintptr) unsafe.Pointer { p, err := mmap(v, n, _PROT_NONE, _MAP_ANON|_MAP_PRIVATE, mmapFD, 0) if err != 0 { return nil } return p } func sysMap(v unsafe.Pointer, n uintptr, sysStat *sysMemStat) { sysStat.add(int64(n)) if GOOS == "aix" { // AIX does not allow mapping a range that is already mapped. // So always unmap first even if it is already unmapped. munmap(v, n) } p, err := mmap(v, n, _PROT_READ|_PROT_WRITE, _MAP_ANON|_MAP_FIXED|_MAP_PRIVATE, mmapFD, 0) if err == _ENOMEM { throw("runtime: out of memory") } if p != v || err != 0 { throw("runtime: cannot map pages in arena address space") } }