123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471 |
- /*
- * Copyright 2011-present Facebook, Inc.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
- /**
- * Improved thread local storage for non-trivial types (similar speed as
- * pthread_getspecific but only consumes a single pthread_key_t, and 4x faster
- * than boost::thread_specific_ptr).
- *
- * Also includes an accessor interface to walk all the thread local child
- * objects of a parent. accessAllThreads() initializes an accessor which holds
- * a global lock *that blocks all creation and destruction of ThreadLocal
- * objects with the same Tag* and can be used as an iterable container.
- * accessAllThreads() can race with destruction of thread-local elements. We
- * provide a strict mode which is dangerous because it requires the access lock
- * to be held while destroying thread-local elements which could cause
- * deadlocks. We gate this mode behind the AccessModeStrict template parameter.
- *
- * Intended use is for frequent write, infrequent read data access patterns such
- * as counters.
- *
- * There are two classes here - ThreadLocal and ThreadLocalPtr. ThreadLocalPtr
- * has semantics similar to boost::thread_specific_ptr. ThreadLocal is a thin
- * wrapper around ThreadLocalPtr that manages allocation automatically.
- *
- * @author Spencer Ahrens (sahrens)
- */
- #pragma once
- #include <iterator>
- #include <type_traits>
- #include <utility>
- #include <folly/Likely.h>
- #include <folly/Portability.h>
- #include <folly/ScopeGuard.h>
- #include <folly/SharedMutex.h>
- #include <folly/detail/ThreadLocalDetail.h>
- namespace folly {
- template <class T, class Tag, class AccessMode>
- class ThreadLocalPtr;
- template <class T, class Tag = void, class AccessMode = void>
- class ThreadLocal {
- public:
- constexpr ThreadLocal() : constructor_([]() { return new T(); }) {}
- template <
- typename F,
- _t<std::enable_if<is_invocable_r<T*, F>::value, int>> = 0>
- explicit ThreadLocal(F&& constructor)
- : constructor_(std::forward<F>(constructor)) {}
- FOLLY_ALWAYS_INLINE FOLLY_ATTR_VISIBILITY_HIDDEN T* get() const {
- auto const ptr = tlp_.get();
- return FOLLY_LIKELY(!!ptr) ? ptr : makeTlp();
- }
- T* operator->() const {
- return get();
- }
- T& operator*() const {
- return *get();
- }
- void reset(T* newPtr = nullptr) {
- tlp_.reset(newPtr);
- }
- typedef typename ThreadLocalPtr<T, Tag, AccessMode>::Accessor Accessor;
- Accessor accessAllThreads() const {
- return tlp_.accessAllThreads();
- }
- // movable
- ThreadLocal(ThreadLocal&&) = default;
- ThreadLocal& operator=(ThreadLocal&&) = default;
- private:
- // non-copyable
- ThreadLocal(const ThreadLocal&) = delete;
- ThreadLocal& operator=(const ThreadLocal&) = delete;
- FOLLY_NOINLINE T* makeTlp() const {
- auto const ptr = constructor_();
- tlp_.reset(ptr);
- return ptr;
- }
- mutable ThreadLocalPtr<T, Tag, AccessMode> tlp_;
- std::function<T*()> constructor_;
- };
- /*
- * The idea here is that __thread is faster than pthread_getspecific, so we
- * keep a __thread array of pointers to objects (ThreadEntry::elements) where
- * each array has an index for each unique instance of the ThreadLocalPtr
- * object. Each ThreadLocalPtr object has a unique id that is an index into
- * these arrays so we can fetch the correct object from thread local storage
- * very efficiently.
- *
- * In order to prevent unbounded growth of the id space and thus huge
- * ThreadEntry::elements, arrays, for example due to continuous creation and
- * destruction of ThreadLocalPtr objects, we keep a set of all active
- * instances. When an instance is destroyed we remove it from the active
- * set and insert the id into freeIds_ for reuse. These operations require a
- * global mutex, but only happen at construction and destruction time.
- *
- * We use a single global pthread_key_t per Tag to manage object destruction and
- * memory cleanup upon thread exit because there is a finite number of
- * pthread_key_t's available per machine.
- *
- * NOTE: Apple platforms don't support the same semantics for __thread that
- * Linux does (and it's only supported at all on i386). For these, use
- * pthread_setspecific()/pthread_getspecific() for the per-thread
- * storage. Windows (MSVC and GCC) does support the same semantics
- * with __declspec(thread)
- */
- template <class T, class Tag = void, class AccessMode = void>
- class ThreadLocalPtr {
- private:
- typedef threadlocal_detail::StaticMeta<Tag, AccessMode> StaticMeta;
- public:
- constexpr ThreadLocalPtr() : id_() {}
- ThreadLocalPtr(ThreadLocalPtr&& other) noexcept : id_(std::move(other.id_)) {}
- ThreadLocalPtr& operator=(ThreadLocalPtr&& other) {
- assert(this != &other);
- destroy();
- id_ = std::move(other.id_);
- return *this;
- }
- ~ThreadLocalPtr() {
- destroy();
- }
- T* get() const {
- threadlocal_detail::ElementWrapper& w = StaticMeta::get(&id_);
- return static_cast<T*>(w.ptr);
- }
- T* operator->() const {
- return get();
- }
- T& operator*() const {
- return *get();
- }
- T* release() {
- threadlocal_detail::ElementWrapper& w = StaticMeta::get(&id_);
- return static_cast<T*>(w.release());
- }
- void reset(T* newPtr = nullptr) {
- auto guard = makeGuard([&] { delete newPtr; });
- threadlocal_detail::ElementWrapper* w = &StaticMeta::get(&id_);
- w->dispose(TLPDestructionMode::THIS_THREAD);
- // need to get a new ptr since the
- // ThreadEntry::elements array can be reallocated
- w = &StaticMeta::get(&id_);
- w->cleanup();
- guard.dismiss();
- w->set(newPtr);
- }
- explicit operator bool() const {
- return get() != nullptr;
- }
- /**
- * reset() that transfers ownership from a smart pointer
- */
- template <
- typename SourceT,
- typename Deleter,
- typename = typename std::enable_if<
- std::is_convertible<SourceT*, T*>::value>::type>
- void reset(std::unique_ptr<SourceT, Deleter> source) {
- auto deleter = [delegate = source.get_deleter()](
- T* ptr, TLPDestructionMode) { delegate(ptr); };
- reset(source.release(), deleter);
- }
- /**
- * reset() that transfers ownership from a smart pointer with the default
- * deleter
- */
- template <
- typename SourceT,
- typename = typename std::enable_if<
- std::is_convertible<SourceT*, T*>::value>::type>
- void reset(std::unique_ptr<SourceT> source) {
- reset(source.release());
- }
- /**
- * reset() with a custom deleter:
- * deleter(T* ptr, TLPDestructionMode mode)
- * "mode" is ALL_THREADS if we're destructing this ThreadLocalPtr (and thus
- * deleting pointers for all threads), and THIS_THREAD if we're only deleting
- * the member for one thread (because of thread exit or reset()).
- * Invoking the deleter must not throw.
- */
- template <class Deleter>
- void reset(T* newPtr, const Deleter& deleter) {
- auto guard = makeGuard([&] {
- if (newPtr) {
- deleter(newPtr, TLPDestructionMode::THIS_THREAD);
- }
- });
- threadlocal_detail::ElementWrapper* w = &StaticMeta::get(&id_);
- w->dispose(TLPDestructionMode::THIS_THREAD);
- // need to get a new ptr since the
- // ThreadEntry::elements array can be reallocated
- w = &StaticMeta::get(&id_);
- w->cleanup();
- guard.dismiss();
- w->set(newPtr, deleter);
- }
- // Holds a global lock for iteration through all thread local child objects.
- // Can be used as an iterable container.
- // Use accessAllThreads() to obtain one.
- class Accessor {
- friend class ThreadLocalPtr<T, Tag, AccessMode>;
- threadlocal_detail::StaticMetaBase& meta_;
- SharedMutex* accessAllThreadsLock_;
- std::mutex* lock_;
- uint32_t id_;
- public:
- class Iterator;
- friend class Iterator;
- // The iterators obtained from Accessor are bidirectional iterators.
- class Iterator {
- friend class Accessor;
- const Accessor* accessor_;
- threadlocal_detail::ThreadEntryNode* e_;
- void increment() {
- e_ = e_->getNext();
- incrementToValid();
- }
- void decrement() {
- e_ = e_->getPrev();
- decrementToValid();
- }
- const T& dereference() const {
- return *static_cast<T*>(
- e_->getThreadEntry()->elements[accessor_->id_].ptr);
- }
- T& dereference() {
- return *static_cast<T*>(
- e_->getThreadEntry()->elements[accessor_->id_].ptr);
- }
- bool equal(const Iterator& other) const {
- return (accessor_->id_ == other.accessor_->id_ && e_ == other.e_);
- }
- explicit Iterator(const Accessor* accessor)
- : accessor_(accessor),
- e_(&accessor_->meta_.head_.elements[accessor_->id_].node) {}
- // we just need to check the ptr since it can be set to nullptr
- // even if the entry is part of the list
- bool valid() const {
- return (e_->getThreadEntry()->elements[accessor_->id_].ptr);
- }
- void incrementToValid() {
- for (; e_ != &accessor_->meta_.head_.elements[accessor_->id_].node &&
- !valid();
- e_ = e_->getNext()) {
- }
- }
- void decrementToValid() {
- for (; e_ != &accessor_->meta_.head_.elements[accessor_->id_].node &&
- !valid();
- e_ = e_->getPrev()) {
- }
- }
- public:
- using difference_type = ssize_t;
- using value_type = T;
- using reference = T const&;
- using pointer = T const*;
- using iterator_category = std::bidirectional_iterator_tag;
- Iterator& operator++() {
- increment();
- return *this;
- }
- Iterator& operator++(int) {
- Iterator copy(*this);
- increment();
- return copy;
- }
- Iterator& operator--() {
- decrement();
- return *this;
- }
- Iterator& operator--(int) {
- Iterator copy(*this);
- decrement();
- return copy;
- }
- T& operator*() {
- return dereference();
- }
- T const& operator*() const {
- return dereference();
- }
- T* operator->() {
- return &dereference();
- }
- T const* operator->() const {
- return &dereference();
- }
- bool operator==(Iterator const& rhs) const {
- return equal(rhs);
- }
- bool operator!=(Iterator const& rhs) const {
- return !equal(rhs);
- }
- };
- ~Accessor() {
- release();
- }
- Iterator begin() const {
- return ++Iterator(this);
- }
- Iterator end() const {
- return Iterator(this);
- }
- Accessor(const Accessor&) = delete;
- Accessor& operator=(const Accessor&) = delete;
- Accessor(Accessor&& other) noexcept
- : meta_(other.meta_),
- accessAllThreadsLock_(other.accessAllThreadsLock_),
- lock_(other.lock_),
- id_(other.id_) {
- other.id_ = 0;
- other.accessAllThreadsLock_ = nullptr;
- other.lock_ = nullptr;
- }
- Accessor& operator=(Accessor&& other) noexcept {
- // Each Tag has its own unique meta, and accessors with different Tags
- // have different types. So either *this is empty, or this and other
- // have the same tag. But if they have the same tag, they have the same
- // meta (and lock), so they'd both hold the lock at the same time,
- // which is impossible, which leaves only one possible scenario --
- // *this is empty. Assert it.
- assert(&meta_ == &other.meta_);
- assert(lock_ == nullptr);
- using std::swap;
- swap(accessAllThreadsLock_, other.accessAllThreadsLock_);
- swap(lock_, other.lock_);
- swap(id_, other.id_);
- }
- Accessor()
- : meta_(threadlocal_detail::StaticMeta<Tag, AccessMode>::instance()),
- accessAllThreadsLock_(nullptr),
- lock_(nullptr),
- id_(0) {}
- private:
- explicit Accessor(uint32_t id)
- : meta_(threadlocal_detail::StaticMeta<Tag, AccessMode>::instance()),
- accessAllThreadsLock_(&meta_.accessAllThreadsLock_),
- lock_(&meta_.lock_) {
- accessAllThreadsLock_->lock();
- lock_->lock();
- id_ = id;
- }
- void release() {
- if (lock_) {
- lock_->unlock();
- DCHECK(accessAllThreadsLock_ != nullptr);
- accessAllThreadsLock_->unlock();
- id_ = 0;
- lock_ = nullptr;
- accessAllThreadsLock_ = nullptr;
- }
- }
- };
- // accessor allows a client to iterate through all thread local child
- // elements of this ThreadLocal instance. Holds a global lock for each <Tag>
- Accessor accessAllThreads() const {
- static_assert(
- !std::is_same<Tag, void>::value,
- "Must use a unique Tag to use the accessAllThreads feature");
- return Accessor(id_.getOrAllocate(StaticMeta::instance()));
- }
- private:
- void destroy() {
- StaticMeta::instance().destroy(&id_);
- }
- // non-copyable
- ThreadLocalPtr(const ThreadLocalPtr&) = delete;
- ThreadLocalPtr& operator=(const ThreadLocalPtr&) = delete;
- mutable typename StaticMeta::EntryID id_;
- };
- namespace threadlocal_detail {
- template <typename>
- struct static_meta_of;
- template <typename T, typename Tag, typename AccessMode>
- struct static_meta_of<ThreadLocalPtr<T, Tag, AccessMode>> {
- using type = StaticMeta<Tag, AccessMode>;
- };
- template <typename T, typename Tag, typename AccessMode>
- struct static_meta_of<ThreadLocal<T, Tag, AccessMode>> {
- using type = StaticMeta<Tag, AccessMode>;
- };
- } // namespace threadlocal_detail
- } // namespace folly
|