123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283 |
- /*
- * Copyright 2018-present Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- #include <folly/experimental/JemallocHugePageAllocator.h>
- #include <folly/portability/String.h>
- #include <glog/logging.h>
- #include <sstream>
- #if defined(MADV_HUGEPAGE) && defined(FOLLY_USE_JEMALLOC) && !FOLLY_SANITIZE
- #include <jemalloc/jemalloc.h>
- #if (JEMALLOC_VERSION_MAJOR >= 5)
- #define FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED 1
- bool folly::JemallocHugePageAllocator::hugePagesSupported{true};
- #endif
- #endif // defined(FOLLY_HAVE_LIBJEMALLOC) && !FOLLY_SANITIZE
- #ifndef FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED
- // Some mocks when jemalloc.h is not included or version too old
- // or when the system does not support the MADV_HUGEPAGE madvise flag
- #undef MALLOCX_ARENA
- #undef MALLOCX_TCACHE_NONE
- #undef MADV_HUGEPAGE
- #define MALLOCX_ARENA(x) 0
- #define MALLOCX_TCACHE_NONE 0
- #define MADV_HUGEPAGE 0
- typedef struct extent_hooks_s extent_hooks_t;
- typedef void*(extent_alloc_t)(
- extent_hooks_t*,
- void*,
- size_t,
- size_t,
- bool*,
- bool*,
- unsigned);
- struct extent_hooks_s {
- extent_alloc_t* alloc;
- };
- bool folly::JemallocHugePageAllocator::hugePagesSupported{false};
- #endif // FOLLY_JEMALLOC_HUGE_PAGE_ALLOCATOR_SUPPORTED
- namespace folly {
- namespace {
- static void print_error(int err, const char* msg) {
- int cur_errno = std::exchange(errno, err);
- PLOG(ERROR) << msg;
- errno = cur_errno;
- }
- class HugePageArena {
- public:
- int init(int nr_pages);
- void* reserve(size_t size, size_t alignment);
- bool addressInArena(void* address) {
- uintptr_t addr = reinterpret_cast<uintptr_t>(address);
- return addr >= start_ && addr < end_;
- }
- size_t freeSpace() {
- return end_ - freePtr_;
- }
- private:
- static void* allocHook(
- extent_hooks_t* extent,
- void* new_addr,
- size_t size,
- size_t alignment,
- bool* zero,
- bool* commit,
- unsigned arena_ind);
- uintptr_t start_{0};
- uintptr_t end_{0};
- uintptr_t freePtr_{0};
- extent_alloc_t* originalAlloc_{nullptr};
- extent_hooks_t extentHooks_;
- };
- constexpr size_t kHugePageSize = 2 * 1024 * 1024;
- // Singleton arena instance
- static HugePageArena arena;
- template <typename T, typename U>
- static inline T align_up(T val, U alignment) {
- DCHECK((alignment & (alignment - 1)) == 0);
- return (val + alignment - 1) & ~(alignment - 1);
- }
- // mmap enough memory to hold the aligned huge pages, then use madvise
- // to get huge pages. Note that this is only a hint and is not guaranteed
- // to be honoured. Check /proc/<pid>/smaps to verify!
- static uintptr_t map_pages(size_t nr_pages) {
- // Initial mmapped area is large enough to contain the aligned huge pages
- size_t alloc_size = nr_pages * kHugePageSize;
- void* p = mmap(
- nullptr,
- alloc_size + kHugePageSize,
- PROT_READ | PROT_WRITE,
- MAP_PRIVATE | MAP_ANONYMOUS,
- -1,
- 0);
- if (p == MAP_FAILED) {
- return 0;
- }
- // Aligned start address
- uintptr_t first_page = align_up((uintptr_t)p, kHugePageSize);
- // Unmap left-over 4k pages
- munmap(p, first_page - (uintptr_t)p);
- munmap(
- (void*)(first_page + alloc_size),
- kHugePageSize - (first_page - (uintptr_t)p));
- // Tell the kernel to please give us huge pages for this range
- madvise((void*)first_page, kHugePageSize * nr_pages, MADV_HUGEPAGE);
- LOG(INFO) << nr_pages << " huge pages at " << (void*)first_page;
- return first_page;
- }
- void* HugePageArena::allocHook(
- extent_hooks_t* extent,
- void* new_addr,
- size_t size,
- size_t alignment,
- bool* zero,
- bool* commit,
- unsigned arena_ind) {
- DCHECK((size & (size - 1)) == 0);
- void* res = nullptr;
- if (new_addr == nullptr) {
- res = arena.reserve(size, alignment);
- }
- LOG(INFO) << "Extent request of size " << size << " alignment " << alignment
- << " = " << res << " (" << arena.freeSpace() << " bytes free)";
- if (res == nullptr) {
- LOG_IF(WARNING, new_addr != nullptr) << "Explicit address not supported";
- res = arena.originalAlloc_(
- extent, new_addr, size, alignment, zero, commit, arena_ind);
- } else {
- if (*zero) {
- bzero(res, size);
- }
- *commit = true;
- }
- return res;
- }
- int HugePageArena::init(int nr_pages) {
- DCHECK(start_ == 0);
- DCHECK(usingJEMalloc());
- unsigned arena_index;
- size_t len = sizeof(arena_index);
- if (auto ret = mallctl("arenas.create", &arena_index, &len, nullptr, 0)) {
- print_error(ret, "Unable to create arena");
- return 0;
- }
- // Set grow retained limit to stop jemalloc from
- // forever increasing the requested size after failed allocations.
- // Normally jemalloc asks for maps of increasing size in order to avoid
- // hitting the limit of allowed mmaps per process.
- // Since this arena is backed by a single mmap and is using huge pages,
- // this is not a concern here.
- // TODO: Support growth of the huge page arena.
- size_t mib[3];
- size_t miblen = sizeof(mib) / sizeof(size_t);
- std::ostringstream rtl_key;
- rtl_key << "arena." << arena_index << ".retain_grow_limit";
- if (auto ret = mallctlnametomib(rtl_key.str().c_str(), mib, &miblen)) {
- print_error(ret, "Unable to read growth limit");
- return 0;
- }
- size_t grow_retained_limit = kHugePageSize;
- mib[1] = arena_index;
- if (auto ret = mallctlbymib(
- mib,
- miblen,
- nullptr,
- nullptr,
- &grow_retained_limit,
- sizeof(grow_retained_limit))) {
- print_error(ret, "Unable to set growth limit");
- return 0;
- }
- std::ostringstream hooks_key;
- hooks_key << "arena." << arena_index << ".extent_hooks";
- extent_hooks_t* hooks;
- len = sizeof(hooks);
- // Read the existing hooks
- if (auto ret = mallctl(hooks_key.str().c_str(), &hooks, &len, nullptr, 0)) {
- print_error(ret, "Unable to get the hooks");
- return 0;
- }
- originalAlloc_ = hooks->alloc;
- // Set the custom hook
- extentHooks_ = *hooks;
- extentHooks_.alloc = &allocHook;
- extent_hooks_t* new_hooks = &extentHooks_;
- if (auto ret = mallctl(
- hooks_key.str().c_str(),
- nullptr,
- nullptr,
- &new_hooks,
- sizeof(new_hooks))) {
- print_error(ret, "Unable to set the hooks");
- return 0;
- }
- start_ = freePtr_ = map_pages(nr_pages);
- if (start_ == 0) {
- return false;
- }
- end_ = start_ + (nr_pages * kHugePageSize);
- return MALLOCX_ARENA(arena_index) | MALLOCX_TCACHE_NONE;
- }
- void* HugePageArena::reserve(size_t size, size_t alignment) {
- VLOG(1) << "Reserve: " << size << " alignemnt " << alignment;
- uintptr_t res = align_up(freePtr_, alignment);
- uintptr_t newFreePtr = res + size;
- if (newFreePtr > end_) {
- LOG(WARNING) << "Request of size " << size << " denied: " << freeSpace()
- << " bytes available - not backed by huge pages";
- return nullptr;
- }
- freePtr_ = newFreePtr;
- return reinterpret_cast<void*>(res);
- }
- } // namespace
- int JemallocHugePageAllocator::flags_{0};
- bool JemallocHugePageAllocator::init(int nr_pages) {
- if (!usingJEMalloc()) {
- LOG(ERROR) << "Not linked with jemalloc?";
- hugePagesSupported = false;
- }
- if (hugePagesSupported) {
- if (flags_ == 0) {
- flags_ = arena.init(nr_pages);
- } else {
- LOG(WARNING) << "Already initialized";
- }
- } else {
- LOG(WARNING) << "Huge Page Allocator not supported";
- }
- return flags_ != 0;
- }
- size_t JemallocHugePageAllocator::freeSpace() {
- return arena.freeSpace();
- }
- bool JemallocHugePageAllocator::addressInArena(void* address) {
- return arena.addressInArena(address);
- }
- } // namespace folly
|