1// Copyright 2017 The Abseil Authors.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14//
15// -----------------------------------------------------------------------------
16// File: call_once.h
17// -----------------------------------------------------------------------------
18//
19// This header file provides an Abseil version of `std::call_once` for invoking
20// a given function at most once, across all threads. This Abseil version is
21// faster than the C++11 version and incorporates the C++17 argument-passing
22// fix, so that (for example) non-const references may be passed to the invoked
23// function.
24
25#ifndef ABSL_BASE_CALL_ONCE_H_
26#define ABSL_BASE_CALL_ONCE_H_
27
28#include <algorithm>
29#include <atomic>
30#include <cstdint>
31#include <type_traits>
32#include <utility>
33
34#include "absl/base/attributes.h"
35#include "absl/base/config.h"
36#include "absl/base/internal/invoke.h"
37#include "absl/base/internal/low_level_scheduling.h"
38#include "absl/base/internal/raw_logging.h"
39#include "absl/base/internal/scheduling_mode.h"
40#include "absl/base/internal/spinlock_wait.h"
41#include "absl/base/macros.h"
42#include "absl/base/nullability.h"
43#include "absl/base/optimization.h"
44#include "absl/base/port.h"
45
46namespace absl {
47ABSL_NAMESPACE_BEGIN
48
49class once_flag;
50
51namespace base_internal {
52absl::Nonnull<std::atomic<uint32_t>*> ControlWord(
53 absl::Nonnull<absl::once_flag*> flag);
54} // namespace base_internal
55
56// call_once()
57//
58// For all invocations using a given `once_flag`, invokes a given `fn` exactly
59// once across all threads. The first call to `call_once()` with a particular
60// `once_flag` argument (that does not throw an exception) will run the
61// specified function with the provided `args`; other calls with the same
62// `once_flag` argument will not run the function, but will wait
63// for the provided function to finish running (if it is still running).
64//
65// This mechanism provides a safe, simple, and fast mechanism for one-time
66// initialization in a multi-threaded process.
67//
68// Example:
69//
70// class MyInitClass {
71// public:
72// ...
73// mutable absl::once_flag once_;
74//
75// MyInitClass* init() const {
76// absl::call_once(once_, &MyInitClass::Init, this);
77// return ptr_;
78// }
79//
80template <typename Callable, typename... Args>
81void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args);
82
83// once_flag
84//
85// Objects of this type are used to distinguish calls to `call_once()` and
86// ensure the provided function is only invoked once across all threads. This
87// type is not copyable or movable. However, it has a `constexpr`
88// constructor, and is safe to use as a namespace-scoped global variable.
89class once_flag {
90 public:
91 constexpr once_flag() : control_(0) {}
92 once_flag(const once_flag&) = delete;
93 once_flag& operator=(const once_flag&) = delete;
94
95 private:
96 friend absl::Nonnull<std::atomic<uint32_t>*> base_internal::ControlWord(
97 absl::Nonnull<once_flag*> flag);
98 std::atomic<uint32_t> control_;
99};
100
101//------------------------------------------------------------------------------
102// End of public interfaces.
103// Implementation details follow.
104//------------------------------------------------------------------------------
105
106namespace base_internal {
107
108// Like call_once, but uses KERNEL_ONLY scheduling. Intended to be used to
109// initialize entities used by the scheduler implementation.
110template <typename Callable, typename... Args>
111void LowLevelCallOnce(absl::Nonnull<absl::once_flag*> flag, Callable&& fn,
112 Args&&... args);
113
114// Disables scheduling while on stack when scheduling mode is non-cooperative.
115// No effect for cooperative scheduling modes.
116class SchedulingHelper {
117 public:
118 explicit SchedulingHelper(base_internal::SchedulingMode mode) : mode_(mode) {
119 if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) {
120 guard_result_ = base_internal::SchedulingGuard::DisableRescheduling();
121 }
122 }
123
124 ~SchedulingHelper() {
125 if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) {
126 base_internal::SchedulingGuard::EnableRescheduling(guard_result_);
127 }
128 }
129
130 private:
131 base_internal::SchedulingMode mode_;
132 bool guard_result_ = false;
133};
134
135// Bit patterns for call_once state machine values. Internal implementation
136// detail, not for use by clients.
137//
138// The bit patterns are arbitrarily chosen from unlikely values, to aid in
139// debugging. However, kOnceInit must be 0, so that a zero-initialized
140// once_flag will be valid for immediate use.
141enum {
142 kOnceInit = 0,
143 kOnceRunning = 0x65C2937B,
144 kOnceWaiter = 0x05A308D2,
145 // A very small constant is chosen for kOnceDone so that it fit in a single
146 // compare with immediate instruction for most common ISAs. This is verified
147 // for x86, POWER and ARM.
148 kOnceDone = 221, // Random Number
149};
150
151template <typename Callable, typename... Args>
152 void
153 CallOnceImpl(absl::Nonnull<std::atomic<uint32_t>*> control,
154 base_internal::SchedulingMode scheduling_mode, Callable&& fn,
155 Args&&... args) {
156#ifndef NDEBUG
157 {
158 uint32_t old_control = control->load(std::memory_order_relaxed);
159 if (old_control != kOnceInit &&
160 old_control != kOnceRunning &&
161 old_control != kOnceWaiter &&
162 old_control != kOnceDone) {
163 ABSL_RAW_LOG(FATAL, "Unexpected value for control word: 0x%lx",
164 static_cast<unsigned long>(old_control)); // NOLINT
165 }
166 }
167#endif // NDEBUG
168 static const base_internal::SpinLockWaitTransition trans[] = {
169 {.from: kOnceInit, .to: kOnceRunning, .done: true},
170 {.from: kOnceRunning, .to: kOnceWaiter, .done: false},
171 {.from: kOnceDone, .to: kOnceDone, .done: true}};
172
173 // Must do this before potentially modifying control word's state.
174 base_internal::SchedulingHelper maybe_disable_scheduling(scheduling_mode);
175 // Short circuit the simplest case to avoid procedure call overhead.
176 // The base_internal::SpinLockWait() call returns either kOnceInit or
177 // kOnceDone. If it returns kOnceDone, it must have loaded the control word
178 // with std::memory_order_acquire and seen a value of kOnceDone.
179 uint32_t old_control = kOnceInit;
180 if (control->compare_exchange_strong(i1&: old_control, i2: kOnceRunning,
181 m: std::memory_order_relaxed) ||
182 base_internal::SpinLockWait(w: control, ABSL_ARRAYSIZE(trans), trans,
183 scheduling_mode) == kOnceInit) {
184 base_internal::invoke(std::forward<Callable>(fn),
185 std::forward<Args>(args)...);
186 old_control =
187 control->exchange(i: base_internal::kOnceDone, m: std::memory_order_release);
188 if (old_control == base_internal::kOnceWaiter) {
189 base_internal::SpinLockWake(w: control, all: true);
190 }
191 } // else *control is already kOnceDone
192}
193
194inline absl::Nonnull<std::atomic<uint32_t>*> ControlWord(
195 absl::Nonnull<once_flag*> flag) {
196 return &flag->control_;
197}
198
199template <typename Callable, typename... Args>
200void LowLevelCallOnce(absl::Nonnull<absl::once_flag*> flag, Callable&& fn,
201 Args&&... args) {
202 std::atomic<uint32_t>* once = base_internal::ControlWord(flag);
203 uint32_t s = once->load(m: std::memory_order_acquire);
204 if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) {
205 base_internal::CallOnceImpl(once, base_internal::SCHEDULE_KERNEL_ONLY,
206 std::forward<Callable>(fn),
207 std::forward<Args>(args)...);
208 }
209}
210
211} // namespace base_internal
212
213template <typename Callable, typename... Args>
214 void
215 call_once(absl::once_flag& flag, Callable&& fn, Args&&... args) {
216 std::atomic<uint32_t>* once = base_internal::ControlWord(flag: &flag);
217 uint32_t s = once->load(m: std::memory_order_acquire);
218 if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) {
219 base_internal::CallOnceImpl(
220 once, base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL,
221 std::forward<Callable>(fn), std::forward<Args>(args)...);
222 }
223}
224
225ABSL_NAMESPACE_END
226} // namespace absl
227
228#endif // ABSL_BASE_CALL_ONCE_H_
229