/* * Copyright (c) Meta Platforms, Inc. and affiliates. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ #pragma once #include #include #include #include #include #include #include #include #include #include namespace folly { namespace access { // locks and unlocks FOLLY_CREATE_MEMBER_INVOKER_SUITE(lock); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_lock); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_lock_for); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_lock_until); FOLLY_CREATE_MEMBER_INVOKER_SUITE(unlock); FOLLY_CREATE_MEMBER_INVOKER_SUITE(lock_shared); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_lock_shared); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_lock_shared_for); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_lock_shared_until); FOLLY_CREATE_MEMBER_INVOKER_SUITE(unlock_shared); FOLLY_CREATE_MEMBER_INVOKER_SUITE(lock_upgrade); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_lock_upgrade); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_lock_upgrade_for); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_lock_upgrade_until); FOLLY_CREATE_MEMBER_INVOKER_SUITE(unlock_upgrade); // transitions FOLLY_CREATE_MEMBER_INVOKER_SUITE(unlock_and_lock_shared); FOLLY_CREATE_MEMBER_INVOKER_SUITE(unlock_and_lock_upgrade); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_unlock_shared_and_lock); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_unlock_shared_and_lock_for); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_unlock_shared_and_lock_until); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_unlock_shared_and_lock_upgrade); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_unlock_shared_and_lock_upgrade_for); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_unlock_shared_and_lock_upgrade_until); FOLLY_CREATE_MEMBER_INVOKER_SUITE(unlock_upgrade_and_lock); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_unlock_upgrade_and_lock); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_unlock_upgrade_and_lock_for); FOLLY_CREATE_MEMBER_INVOKER_SUITE(try_unlock_upgrade_and_lock_until); FOLLY_CREATE_MEMBER_INVOKER_SUITE(unlock_upgrade_and_lock_shared); } // namespace access struct adopt_lock_state_t {}; inline constexpr adopt_lock_state_t adopt_lock_state{}; namespace detail { // A lock base class with a mostly-complete implementation suitable for either // unique, shared, or upgrade lock base classes. However, each particular base // class specific to each lock category must still be its own class to avoid // overly permissive overloads of member and free swap. template class lock_base { public: using mutex_type = Mutex; using state_type = invoke_result_t; static_assert( std::is_same>::value, "state_type, if not void, must be a value type"); static_assert( std::is_void::value || (std::is_nothrow_default_constructible::value && std::is_nothrow_copy_constructible::value && std::is_nothrow_copy_assignable::value && std::is_nothrow_destructible::value), "state_type, if not void, must be noexcept-semiregular"); static_assert( std::is_void::value || std::is_constructible::value, "state_type, if not void, must explicitly convert to bool"); private: static constexpr bool has_state_ = !std::is_void::value; using owner_type = conditional_t; template using if_ = std::enable_if_t; static bool owner_true_(tag_t) noexcept { return true; } static owner_type owner_true_(tag_t) noexcept { return {}; } mutex_type* mutex_{}; owner_type state_{}; public: FOLLY_NODISCARD lock_base() = default; FOLLY_NODISCARD lock_base(lock_base&& that) noexcept : mutex_{std::exchange(that.mutex_, nullptr)}, state_{std::exchange(that.state_, owner_type{})} {} template * = nullptr> FOLLY_NODISCARD lock_base(type_t& mutex, std::adopt_lock_t) : mutex_{std::addressof(mutex)}, state_{owner_true_(tag)} {} template * = nullptr> FOLLY_NODISCARD lock_base( type_t& mutex, std::adopt_lock_t, owner_type const& state) : mutex_{std::addressof(mutex)}, state_{state} { state_ || (check_fail_(), 0); } template * = nullptr> FOLLY_NODISCARD lock_base( type_t& mutex, adopt_lock_state_t, owner_type const& state) : lock_base{mutex, std::adopt_lock, state} {} FOLLY_NODISCARD explicit lock_base(mutex_type& mutex) : mutex_{std::addressof(mutex)} { lock(); } lock_base(mutex_type& mutex, std::defer_lock_t) noexcept : mutex_{std::addressof(mutex)} {} FOLLY_NODISCARD lock_base(mutex_type& mutex, std::try_to_lock_t) : mutex_{std::addressof(mutex)} { try_lock(); } template FOLLY_NODISCARD lock_base( mutex_type& mutex, std::chrono::duration const& timeout) : mutex_{std::addressof(mutex)} { try_lock_for(timeout); } template FOLLY_NODISCARD lock_base( mutex_type& mutex, std::chrono::time_point const& deadline) : mutex_{std::addressof(mutex)} { try_lock_until(deadline); } ~lock_base() { if (owns_lock()) { unlock(); } } lock_base& operator=(lock_base&& that) noexcept { if (owns_lock()) { unlock(); } mutex_ = std::exchange(that.mutex_, nullptr); state_ = std::exchange(that.state_, owner_type{}); return *this; } void lock() { check(); if constexpr (has_state_) { state_ = typename Policy::lock_fn{}(*mutex_); } else { typename Policy::lock_fn{}(*mutex_); state_ = true; } } bool try_lock() { check(); state_ = typename Policy::try_lock_fn{}(*mutex_); return !!state_; } template bool try_lock_for(std::chrono::duration const& timeout) { check(); state_ = typename Policy::try_lock_for_fn{}(*mutex_, timeout); return !!state_; } template bool try_lock_until( std::chrono::time_point const& deadline) { check(); state_ = typename Policy::try_lock_until_fn{}(*mutex_, deadline); return !!state_; } void unlock() { check(); if constexpr (has_state_) { // prohibit unlock to mutate state_ typename Policy::unlock_fn{}(*mutex_, std::as_const(state_)); } else { typename Policy::unlock_fn{}(*mutex_); } state_ = owner_type{}; } mutex_type* release() noexcept { state_ = owner_type{}; return std::exchange(mutex_, nullptr); } mutex_type* mutex() const noexcept { return mutex_; } template = 0> state_type state() const noexcept { return state_; } bool owns_lock() const noexcept { return !!state_; } explicit operator bool() const noexcept { return !!state_; } protected: void swap(lock_base& that) noexcept { std::swap(mutex_, that.mutex_); std::swap(state_, that.state_); } private: template void check() { if (!mutex_ || !state_ == Owns) { check_fail_(); } } template [[noreturn]] FOLLY_NOINLINE void check_fail_() { auto perm = std::errc::operation_not_permitted; auto dead = std::errc::resource_deadlock_would_occur; auto code = !mutex_ || !state_ ? perm : dead; throw_exception(std::make_error_code(code)); } }; template class lock_guard_base : unsafe_for_async_usage_if> { private: using lock_type_ = lock_base; using lock_state_type_ = typename lock_type_::state_type; static constexpr bool has_state_ = !std::is_void::value; using state_type_ = conditional_t; template using if_ = std::enable_if_t; public: using mutex_type = Mutex; lock_guard_base(lock_guard_base const&) = delete; lock_guard_base(lock_guard_base&&) = delete; explicit lock_guard_base(mutex_type& mutex) : lock_{mutex} {} template = 0> lock_guard_base(mutex_type& mutex, std::adopt_lock_t) : lock_{mutex, std::adopt_lock} {} template = 0> lock_guard_base( mutex_type& mutex, std::adopt_lock_t, state_type_ const& state) : lock_{mutex, std::adopt_lock, state} {} template = 0> lock_guard_base( mutex_type& mutex, adopt_lock_state_t, state_type_ const& state) : lock_{mutex, std::adopt_lock, state} {} void operator=(lock_guard_base const&) = delete; void operator=(lock_guard_base&&) = delete; private: lock_type_ lock_; }; struct lock_policy_unique { using lock_fn = access::lock_fn; using try_lock_fn = access::try_lock_fn; using try_lock_for_fn = access::try_lock_for_fn; using try_lock_until_fn = access::try_lock_until_fn; using unlock_fn = access::unlock_fn; }; struct lock_policy_shared { using lock_fn = access::lock_shared_fn; using try_lock_fn = access::try_lock_shared_fn; using try_lock_for_fn = access::try_lock_shared_for_fn; using try_lock_until_fn = access::try_lock_shared_until_fn; using unlock_fn = access::unlock_shared_fn; }; struct lock_policy_upgrade { using lock_fn = access::lock_upgrade_fn; using try_lock_fn = access::try_lock_upgrade_fn; using try_lock_for_fn = access::try_lock_upgrade_for_fn; using try_lock_until_fn = access::try_lock_upgrade_until_fn; using unlock_fn = access::unlock_upgrade_fn; }; template using lock_policy_hybrid = conditional_t< is_invocable_v, lock_policy_shared, lock_policy_unique>; template using lock_base_unique = lock_base; template using lock_base_shared = lock_base; template using lock_base_upgrade = lock_base; template using lock_base_hybrid = lock_base>; } // namespace detail // unique_lock_base // // A lock-holder base which holds exclusive locks, usable with any mutex type. // // Works with both lockable mutex types and lockable-with-state mutex types. // // When defining lockable-with-state mutex types, specialize std::unique_lock // to derive this. See the example with upgrade_lock. // // A lockable-with-state mutex type is signalled by the return type of mutex // member function lock. Members try_lock, try_lock_for, and try_lock_until // all return this type and member unlock accepts this type. template class unique_lock_base : public detail::lock_base_unique { private: using base = detail::lock_base_unique; using self = unique_lock_base; public: using base::base; void swap(self& that) noexcept { base::swap(that); } friend void swap(self& a, self& b) noexcept { a.swap(b); } }; // shared_lock_base // // A lock-holder base which holds shared locks, usable with any shared mutex // type. // // Works with both shared-lockable mutex types and shared-lockable-with-state // mutex types. // // When defining shared-lockable-with-state mutex types, specialize // std::shared_lock to derive this. See the example with upgrade_lock. // // A shared-lockable-with-state mutex type is signalled by the return type of // mutex member function lock_shared. Members try_lock_shared, // try_lock_shared_for, and try_lock_shared_until all return this type and // member unlock_shared accepts this type. Likewise for mutex member // transition functions. template class shared_lock_base : public detail::lock_base_shared { private: using base = detail::lock_base_shared; using self = shared_lock_base; public: using base::base; void swap(self& that) noexcept { base::swap(that); } friend void swap(self& a, self& b) noexcept { a.swap(b); } }; // upgrade_lock_base // // A lock-holder base which holds upgrade locks, usable with any upgrade mutex // type. // // Works with both upgrade-lockable mutex types and upgrade-lockable-with-state // mutex types. // // There are no use-cases except the one below. // // An upgrade-lockable-with-state mutex type is signalled by the return type of // mutex member function lock_upgrade. Members try_lock_upgrade, // try_lock_upgrade_for, and try_lock_upgrade_until all return this type and // member unlock_upgrade accepts this type. Likewise for mutex member // transition functions. template class upgrade_lock_base : public detail::lock_base_upgrade { private: using base = detail::lock_base_upgrade; using self = upgrade_lock_base; public: using base::base; void swap(self& that) noexcept { base::swap(that); } friend void swap(self& a, self& b) noexcept { a.swap(b); } }; // hybrid_lock_base // // A lock-holder base which holds shared locks for shared mutex types or // exclusive locks otherwise. // // See unique_lock_base and shared_lock_base. template class hybrid_lock_base : public detail::lock_base_hybrid { private: using base = detail::lock_base_hybrid; using self = hybrid_lock_base; public: using base::base; void swap(self& that) noexcept { base::swap(that); } friend void swap(self& a, self& b) noexcept { a.swap(b); } }; // unique_lock // // Alias to std::unique_lock. using std::unique_lock; // shared_lock // // Alias to std::shared_lock. using std::shared_lock; // upgrade_lock // // A lock-holder type which holds upgrade locks, usable with any upgrade mutex // type. An upgrade mutex is a shared mutex which supports the upgrade state. // // Works with both upgrade-lockable mutex types and upgrade-lockable-with-state // mutex types. // // Upgrade locks are not useful by themselves; they are primarily useful since // upgrade locks may be transitioned atomically to exclusive locks. This lock- // holder type works with the transition_to_... functions below to facilitate // atomic transition from ugprade lock to exclusive lock. template class upgrade_lock : public upgrade_lock_base { public: using upgrade_lock_base::upgrade_lock_base; }; template explicit upgrade_lock(Mutex&, A const&...) -> upgrade_lock; // hybrid_lock // // A lock-holder type which holds shared locks for shared mutex types or // exclusive locks otherwise. // // See unique_lock and shared_lock. template class hybrid_lock : public hybrid_lock_base { public: using hybrid_lock_base::hybrid_lock_base; }; template explicit hybrid_lock(Mutex&, A const&...) -> hybrid_lock; // lock_guard_base // // A lock-guard which holds exclusive locks, usable with any mutex type. // // Works with both lockable mutex types and lockable-with-state mutex types. // // When defining lockable-with-state mutex types, specialize std::lock_guard // to derive this. template class unique_lock_guard_base : public detail::lock_guard_base { private: using base = detail::lock_guard_base; public: using base::base; }; // unique_lock_guard // // Alias to std::lock_guard. template using unique_lock_guard = std::lock_guard; // shared_lock_guard // // A lock-guard which holds shared locks, usable with any shared mutex type. // // Works with both lockable mutex types and lockable-with-state mutex types. template class shared_lock_guard : public detail::lock_guard_base { private: using base = detail::lock_guard_base; public: using base::base; }; // hybrid_lock_guard // // For shared mutex types, effectively shared_lock_guard; otherwise, // effectively unique_lock_guard. template class hybrid_lock_guard : public detail::lock_guard_base> { private: using base = detail::lock_guard_base>; public: using base::base; }; template hybrid_lock_guard(Mutex&, adopt_lock_state_t, S) -> hybrid_lock_guard; } // namespace folly FOLLY_NAMESPACE_STD_BEGIN template unique_lock(Mutex&, folly::adopt_lock_state_t, S) -> unique_lock; template shared_lock(Mutex&, folly::adopt_lock_state_t, S) -> shared_lock; template lock_guard(Mutex&, folly::adopt_lock_state_t, S) -> lock_guard; FOLLY_NAMESPACE_STD_END namespace folly { namespace detail { template using lock_state_type_of_t_ = typename L::state_type; template using lock_state_type_of_t = detected_or_t; template struct transition_lock_result_ { template using apply = invoke_result_t; }; template <> struct transition_lock_result_ { template using apply = invoke_result_t; }; template using transition_lock_result_t_ = typename transition_lock_result_>:: template apply; template auto transition_lock_2_(From& lock, Transition transition, A const&... a) { using FromState = lock_state_type_of_t; if constexpr (std::is_void_v) { // release() may check or mutate mutex state to support the dissociation; // call it before performing the transition. return transition(*lock.release(), a...); } else { auto state = lock.state(); // release() may check or mutate mutex state to support the dissociation; // call it before performing the transition. return transition(*lock.release(), std::move(state), a...); } } template auto transition_lock_1_(From& lock, Transition transition, A const&... a) { using Result = transition_lock_result_t_; if constexpr (std::is_void_v) { return detail::transition_lock_2_(lock, transition, a...), true; } else { return detail::transition_lock_2_(lock, transition, a...); } } template auto transition_lock_0_(From& lock, Transition transition, A const&... a) { using ToState = lock_state_type_of_t; auto& mutex = *lock.mutex(); auto s = detail::transition_lock_1_(lock, transition, a...); if constexpr (std::is_void_v) { return !s ? To{} : To{mutex, std::adopt_lock}; } else { return !s ? To{} : To{mutex, folly::adopt_lock_state, s}; } } template < template class To, template class From, typename Mutex, typename Transition, typename... A> auto transition_lock_(From& lock, Transition transition, A const&... a) { // clang-format off return !lock.mutex() ? To{} : !lock.owns_lock() ? To{*lock.release(), std::defer_lock} : detail::transition_lock_0_>(lock, transition, a...); // clang-format on } template struct transition_lock_policy; template struct transition_lock_policy, shared_lock> { using transition_fn = access::unlock_and_lock_shared_fn; }; template struct transition_lock_policy, upgrade_lock> { using transition_fn = access::unlock_and_lock_upgrade_fn; }; template struct transition_lock_policy, unique_lock> { using try_transition_fn = access::try_unlock_shared_and_lock_fn; using try_transition_for_fn = access::try_unlock_shared_and_lock_for_fn; using try_transition_until_fn = access::try_unlock_shared_and_lock_until_fn; }; template struct transition_lock_policy, upgrade_lock> { using try_transition_fn = access::try_unlock_shared_and_lock_upgrade_fn; using try_transition_for_fn = access::try_unlock_shared_and_lock_upgrade_for_fn; using try_transition_until_fn = access::try_unlock_shared_and_lock_upgrade_until_fn; }; template struct transition_lock_policy, unique_lock> { using transition_fn = access::unlock_upgrade_and_lock_fn; using try_transition_fn = access::try_unlock_upgrade_and_lock_fn; using try_transition_for_fn = access::try_unlock_upgrade_and_lock_for_fn; using try_transition_until_fn = access::try_unlock_upgrade_and_lock_until_fn; }; template struct transition_lock_policy, shared_lock> { using transition_fn = access::unlock_upgrade_and_lock_shared_fn; }; } // namespace detail // transition_lock // // Represents an atomic transition from the from-lock to the to-lock. Waits // unboundedly for the transition to become available. template < template class ToLock, typename Mutex, template class FromLock> ToLock transition_lock(FromLock& lock) { using policy = detail::transition_lock_policy, ToLock>; auto _ = typename policy::transition_fn{}; return detail::transition_lock_(lock, _); } // try_transition_lock // // Represents an atomic transition attempt from the from-lock to the to-lock. // Does not wait if the transition is not immediately available. template < template class ToLock, typename Mutex, template class FromLock> ToLock try_transition_lock(FromLock& lock) { using policy = detail::transition_lock_policy, ToLock>; auto _ = typename policy::try_transition_fn{}; return detail::transition_lock_(lock, _); } // try_transition_lock_for // // Represents an atomic transition attempt from the from-lock to the to-lock // bounded by a timeout. Waits up to the timeout for the transition to become // available. template < template class ToLock, typename Mutex, template class FromLock, typename Rep, typename Period> ToLock try_transition_lock_for( FromLock& lock, std::chrono::duration const& timeout) { using policy = detail::transition_lock_policy, ToLock>; auto _ = typename policy::try_transition_for_fn{}; return detail::transition_lock_(lock, _, timeout); } // try_transition_lock_until // // Represents an atomic transition attempt from the from-lock to the to-lock // bounded by a deadline. Waits up to the deadline for the transition to become // available. template < template class ToLock, typename Mutex, template class FromLock, typename Clock, typename Duration> ToLock try_transition_lock_until( FromLock& lock, std::chrono::time_point const& deadline) { using policy = detail::transition_lock_policy, ToLock>; auto _ = typename policy::try_transition_until_fn{}; return detail::transition_lock_(lock, _, deadline); } // transition_to_shared_lock(unique_lock) // // Wraps mutex member function unlock_and_lock_shared. // // Represents an immediate atomic downgrade transition from exclusive lock to // to shared lock. template shared_lock transition_to_shared_lock(unique_lock& lock) { return transition_lock(lock); } // transition_to_shared_lock(upgrade_lock) // // Wraps mutex member function unlock_upgrade_and_lock_shared. // // Represents an immediate atomic downgrade transition from upgrade lock to // shared lock. template shared_lock transition_to_shared_lock(upgrade_lock& lock) { return transition_lock(lock); } // transition_to_upgrade_lock(unique_lock) // // Wraps mutex member function unlock_and_lock_upgrade. // // Represents an immediate atomic downgrade transition from unique lock to // upgrade lock. template upgrade_lock transition_to_upgrade_lock(unique_lock& lock) { return transition_lock(lock); } // transition_to_unique_lock(upgrade_lock) // // Wraps mutex member function unlock_upgrade_and_lock. // // Represents an eventual atomic upgrade transition from upgrade lock to unique // lock. template unique_lock transition_to_unique_lock(upgrade_lock& lock) { return transition_lock(lock); } // try_transition_to_unique_lock(upgrade_lock) // // Wraps mutex member function try_unlock_upgrade_and_lock. // // Represents an immediate attempted atomic upgrade transition from upgrade // lock to unique lock. template unique_lock try_transition_to_unique_lock(upgrade_lock& lock) { return transition_lock(lock); } // try_transition_to_unique_lock_for(upgrade_lock) // // Wraps mutex member function try_unlock_upgrade_and_lock_for. // // Represents an eventual attempted atomic upgrade transition from upgrade // lock to unique lock. template unique_lock try_transition_to_unique_lock_for( upgrade_lock& lock, std::chrono::duration const& timeout) { return try_transition_lock_for(lock, timeout); } // try_transition_to_unique_lock_until(upgrade_lock) // // Wraps mutex member function try_unlock_upgrade_and_lock_until. // // Represents an eventual attempted atomic upgrade transition from upgrade // lock to unique lock. template unique_lock try_transition_to_unique_lock_until( upgrade_lock& lock, std::chrono::time_point const& deadline) { return try_transition_lock_until(lock, deadline); } // try_transition_to_unique_lock(shared_lock) // // Wraps mutex member function try_unlock_shared_and_lock. // // Represents an immediate attempted atomic upgrade transition from shared // lock to unique lock. template unique_lock try_transition_to_unique_lock(shared_lock& lock) { return try_transition_lock(lock); } // try_transition_to_unique_lock_for(shared_lock) // // Wraps mutex member function try_unlock_shared_and_lock_for. // // Represents an eventual attempted atomic upgrade transition from shared // lock to unique lock. template unique_lock try_transition_to_unique_lock_for( shared_lock& lock, std::chrono::duration const& timeout) { return try_transition_lock_for(lock, timeout); } // try_transition_to_unique_lock_until(shared_lock) // // Wraps mutex member function try_unlock_shared_and_lock_until. // // Represents an eventual attempted atomic upgrade transition from shared // lock to unique lock. template unique_lock try_transition_to_unique_lock_until( shared_lock& lock, std::chrono::time_point const& deadline) { return try_transition_lock_until(lock, deadline); } // try_transition_to_upgrade_lock(shared_lock) // // Wraps mutex member function try_unlock_shared_and_lock_upgrade. // // Represents an immediate attempted atomic upgrade transition from shared // lock to upgrade lock. template upgrade_lock try_transition_to_upgrade_lock(shared_lock& lock) { return try_transition_lock(lock); } // try_transition_to_upgrade_lock_for(shared_lock) // // Wraps mutex member function try_unlock_shared_and_lock_upgrade_for. // // Represents an eventual attempted atomic upgrade transition from shared // lock to upgrade lock. template upgrade_lock try_transition_to_upgrade_lock_for( shared_lock& lock, std::chrono::duration const& timeout) { return try_transition_lock_for(lock, timeout); } // try_transition_to_upgrade_lock_until(shared_lock) // // Wraps mutex member function try_unlock_shared_and_lock_upgrade_until. // // Represents an eventual attempted atomic upgrade transition from shared // lock to upgrade lock. template upgrade_lock try_transition_to_upgrade_lock_until( shared_lock& lock, std::chrono::time_point const& deadline) { return try_transition_lock_until(lock, deadline); } } // namespace folly