1 | // Copyright 2017 The Abseil Authors. |
2 | // |
3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
4 | // you may not use this file except in compliance with the License. |
5 | // You may obtain a copy of the License at |
6 | // |
7 | // https://www.apache.org/licenses/LICENSE-2.0 |
8 | // |
9 | // Unless required by applicable law or agreed to in writing, software |
10 | // distributed under the License is distributed on an "AS IS" BASIS, |
11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
12 | // See the License for the specific language governing permissions and |
13 | // limitations under the License. |
14 | // |
15 | // ----------------------------------------------------------------------------- |
16 | // File: call_once.h |
17 | // ----------------------------------------------------------------------------- |
18 | // |
19 | // This header file provides an Abseil version of `std::call_once` for invoking |
20 | // a given function at most once, across all threads. This Abseil version is |
21 | // faster than the C++11 version and incorporates the C++17 argument-passing |
22 | // fix, so that (for example) non-const references may be passed to the invoked |
23 | // function. |
24 | |
25 | #ifndef ABSL_BASE_CALL_ONCE_H_ |
26 | #define ABSL_BASE_CALL_ONCE_H_ |
27 | |
28 | #include <algorithm> |
29 | #include <atomic> |
30 | #include <cstdint> |
31 | #include <type_traits> |
32 | #include <utility> |
33 | |
34 | #include "absl/base/internal/invoke.h" |
35 | #include "absl/base/internal/low_level_scheduling.h" |
36 | #include "absl/base/internal/raw_logging.h" |
37 | #include "absl/base/internal/scheduling_mode.h" |
38 | #include "absl/base/internal/spinlock_wait.h" |
39 | #include "absl/base/macros.h" |
40 | #include "absl/base/nullability.h" |
41 | #include "absl/base/optimization.h" |
42 | #include "absl/base/port.h" |
43 | |
44 | namespace absl { |
45 | ABSL_NAMESPACE_BEGIN |
46 | |
47 | class once_flag; |
48 | |
49 | namespace base_internal { |
50 | absl::Nonnull<std::atomic<uint32_t>*> ControlWord( |
51 | absl::Nonnull<absl::once_flag*> flag); |
52 | } // namespace base_internal |
53 | |
54 | // call_once() |
55 | // |
56 | // For all invocations using a given `once_flag`, invokes a given `fn` exactly |
57 | // once across all threads. The first call to `call_once()` with a particular |
58 | // `once_flag` argument (that does not throw an exception) will run the |
59 | // specified function with the provided `args`; other calls with the same |
60 | // `once_flag` argument will not run the function, but will wait |
61 | // for the provided function to finish running (if it is still running). |
62 | // |
63 | // This mechanism provides a safe, simple, and fast mechanism for one-time |
64 | // initialization in a multi-threaded process. |
65 | // |
66 | // Example: |
67 | // |
68 | // class MyInitClass { |
69 | // public: |
70 | // ... |
71 | // mutable absl::once_flag once_; |
72 | // |
73 | // MyInitClass* init() const { |
74 | // absl::call_once(once_, &MyInitClass::Init, this); |
75 | // return ptr_; |
76 | // } |
77 | // |
78 | template <typename Callable, typename... Args> |
79 | void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args); |
80 | |
81 | // once_flag |
82 | // |
83 | // Objects of this type are used to distinguish calls to `call_once()` and |
84 | // ensure the provided function is only invoked once across all threads. This |
85 | // type is not copyable or movable. However, it has a `constexpr` |
86 | // constructor, and is safe to use as a namespace-scoped global variable. |
87 | class once_flag { |
88 | public: |
89 | constexpr once_flag() : control_(0) {} |
90 | once_flag(const once_flag&) = delete; |
91 | once_flag& operator=(const once_flag&) = delete; |
92 | |
93 | private: |
94 | friend absl::Nonnull<std::atomic<uint32_t>*> base_internal::ControlWord( |
95 | absl::Nonnull<once_flag*> flag); |
96 | std::atomic<uint32_t> control_; |
97 | }; |
98 | |
99 | //------------------------------------------------------------------------------ |
100 | // End of public interfaces. |
101 | // Implementation details follow. |
102 | //------------------------------------------------------------------------------ |
103 | |
104 | namespace base_internal { |
105 | |
106 | // Like call_once, but uses KERNEL_ONLY scheduling. Intended to be used to |
107 | // initialize entities used by the scheduler implementation. |
108 | template <typename Callable, typename... Args> |
109 | void LowLevelCallOnce(absl::Nonnull<absl::once_flag*> flag, Callable&& fn, |
110 | Args&&... args); |
111 | |
112 | // Disables scheduling while on stack when scheduling mode is non-cooperative. |
113 | // No effect for cooperative scheduling modes. |
114 | class SchedulingHelper { |
115 | public: |
116 | explicit SchedulingHelper(base_internal::SchedulingMode mode) : mode_(mode) { |
117 | if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) { |
118 | guard_result_ = base_internal::SchedulingGuard::DisableRescheduling(); |
119 | } |
120 | } |
121 | |
122 | ~SchedulingHelper() { |
123 | if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) { |
124 | base_internal::SchedulingGuard::EnableRescheduling(guard_result_); |
125 | } |
126 | } |
127 | |
128 | private: |
129 | base_internal::SchedulingMode mode_; |
130 | bool guard_result_ = false; |
131 | }; |
132 | |
133 | // Bit patterns for call_once state machine values. Internal implementation |
134 | // detail, not for use by clients. |
135 | // |
136 | // The bit patterns are arbitrarily chosen from unlikely values, to aid in |
137 | // debugging. However, kOnceInit must be 0, so that a zero-initialized |
138 | // once_flag will be valid for immediate use. |
139 | enum { |
140 | kOnceInit = 0, |
141 | kOnceRunning = 0x65C2937B, |
142 | kOnceWaiter = 0x05A308D2, |
143 | // A very small constant is chosen for kOnceDone so that it fit in a single |
144 | // compare with immediate instruction for most common ISAs. This is verified |
145 | // for x86, POWER and ARM. |
146 | kOnceDone = 221, // Random Number |
147 | }; |
148 | |
149 | template <typename Callable, typename... Args> |
150 | ABSL_ATTRIBUTE_NOINLINE void CallOnceImpl( |
151 | absl::Nonnull<std::atomic<uint32_t>*> control, |
152 | base_internal::SchedulingMode scheduling_mode, Callable&& fn, |
153 | Args&&... args) { |
154 | #ifndef NDEBUG |
155 | { |
156 | uint32_t old_control = control->load(std::memory_order_relaxed); |
157 | if (old_control != kOnceInit && |
158 | old_control != kOnceRunning && |
159 | old_control != kOnceWaiter && |
160 | old_control != kOnceDone) { |
161 | ABSL_RAW_LOG(FATAL, "Unexpected value for control word: 0x%lx" , |
162 | static_cast<unsigned long>(old_control)); // NOLINT |
163 | } |
164 | } |
165 | #endif // NDEBUG |
166 | static const base_internal::SpinLockWaitTransition trans[] = { |
167 | {.from: kOnceInit, .to: kOnceRunning, .done: true}, |
168 | {.from: kOnceRunning, .to: kOnceWaiter, .done: false}, |
169 | {.from: kOnceDone, .to: kOnceDone, .done: true}}; |
170 | |
171 | // Must do this before potentially modifying control word's state. |
172 | base_internal::SchedulingHelper maybe_disable_scheduling(scheduling_mode); |
173 | // Short circuit the simplest case to avoid procedure call overhead. |
174 | // The base_internal::SpinLockWait() call returns either kOnceInit or |
175 | // kOnceDone. If it returns kOnceDone, it must have loaded the control word |
176 | // with std::memory_order_acquire and seen a value of kOnceDone. |
177 | uint32_t old_control = kOnceInit; |
178 | if (control->compare_exchange_strong(i1&: old_control, i2: kOnceRunning, |
179 | m: std::memory_order_relaxed) || |
180 | base_internal::SpinLockWait(w: control, ABSL_ARRAYSIZE(trans), trans, |
181 | scheduling_mode) == kOnceInit) { |
182 | base_internal::invoke(std::forward<Callable>(fn), |
183 | std::forward<Args>(args)...); |
184 | old_control = |
185 | control->exchange(i: base_internal::kOnceDone, m: std::memory_order_release); |
186 | if (old_control == base_internal::kOnceWaiter) { |
187 | base_internal::SpinLockWake(w: control, all: true); |
188 | } |
189 | } // else *control is already kOnceDone |
190 | } |
191 | |
192 | inline absl::Nonnull<std::atomic<uint32_t>*> ControlWord( |
193 | absl::Nonnull<once_flag*> flag) { |
194 | return &flag->control_; |
195 | } |
196 | |
197 | template <typename Callable, typename... Args> |
198 | void LowLevelCallOnce(absl::Nonnull<absl::once_flag*> flag, Callable&& fn, |
199 | Args&&... args) { |
200 | std::atomic<uint32_t>* once = base_internal::ControlWord(flag); |
201 | uint32_t s = once->load(m: std::memory_order_acquire); |
202 | if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) { |
203 | base_internal::CallOnceImpl(once, base_internal::SCHEDULE_KERNEL_ONLY, |
204 | std::forward<Callable>(fn), |
205 | std::forward<Args>(args)...); |
206 | } |
207 | } |
208 | |
209 | } // namespace base_internal |
210 | |
211 | template <typename Callable, typename... Args> |
212 | void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args) { |
213 | std::atomic<uint32_t>* once = base_internal::ControlWord(flag: &flag); |
214 | uint32_t s = once->load(m: std::memory_order_acquire); |
215 | if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) { |
216 | base_internal::CallOnceImpl( |
217 | once, base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL, |
218 | std::forward<Callable>(fn), std::forward<Args>(args)...); |
219 | } |
220 | } |
221 | |
222 | ABSL_NAMESPACE_END |
223 | } // namespace absl |
224 | |
225 | #endif // ABSL_BASE_CALL_ONCE_H_ |
226 | |