1// Copyright 2017 The Abseil Authors.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7// https://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14//
15// -----------------------------------------------------------------------------
16// File: call_once.h
17// -----------------------------------------------------------------------------
18//
19// This header file provides an Abseil version of `std::call_once` for invoking
20// a given function at most once, across all threads. This Abseil version is
21// faster than the C++11 version and incorporates the C++17 argument-passing
22// fix, so that (for example) non-const references may be passed to the invoked
23// function.
24
25#ifndef ABSL_BASE_CALL_ONCE_H_
26#define ABSL_BASE_CALL_ONCE_H_
27
28#include <algorithm>
29#include <atomic>
30#include <cstdint>
31#include <type_traits>
32#include <utility>
33
34#include "absl/base/internal/invoke.h"
35#include "absl/base/internal/low_level_scheduling.h"
36#include "absl/base/internal/raw_logging.h"
37#include "absl/base/internal/scheduling_mode.h"
38#include "absl/base/internal/spinlock_wait.h"
39#include "absl/base/macros.h"
40#include "absl/base/nullability.h"
41#include "absl/base/optimization.h"
42#include "absl/base/port.h"
43
44namespace absl {
45ABSL_NAMESPACE_BEGIN
46
47class once_flag;
48
49namespace base_internal {
50absl::Nonnull<std::atomic<uint32_t>*> ControlWord(
51 absl::Nonnull<absl::once_flag*> flag);
52} // namespace base_internal
53
54// call_once()
55//
56// For all invocations using a given `once_flag`, invokes a given `fn` exactly
57// once across all threads. The first call to `call_once()` with a particular
58// `once_flag` argument (that does not throw an exception) will run the
59// specified function with the provided `args`; other calls with the same
60// `once_flag` argument will not run the function, but will wait
61// for the provided function to finish running (if it is still running).
62//
63// This mechanism provides a safe, simple, and fast mechanism for one-time
64// initialization in a multi-threaded process.
65//
66// Example:
67//
68// class MyInitClass {
69// public:
70// ...
71// mutable absl::once_flag once_;
72//
73// MyInitClass* init() const {
74// absl::call_once(once_, &MyInitClass::Init, this);
75// return ptr_;
76// }
77//
78template <typename Callable, typename... Args>
79void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args);
80
81// once_flag
82//
83// Objects of this type are used to distinguish calls to `call_once()` and
84// ensure the provided function is only invoked once across all threads. This
85// type is not copyable or movable. However, it has a `constexpr`
86// constructor, and is safe to use as a namespace-scoped global variable.
87class once_flag {
88 public:
89 constexpr once_flag() : control_(0) {}
90 once_flag(const once_flag&) = delete;
91 once_flag& operator=(const once_flag&) = delete;
92
93 private:
94 friend absl::Nonnull<std::atomic<uint32_t>*> base_internal::ControlWord(
95 absl::Nonnull<once_flag*> flag);
96 std::atomic<uint32_t> control_;
97};
98
99//------------------------------------------------------------------------------
100// End of public interfaces.
101// Implementation details follow.
102//------------------------------------------------------------------------------
103
104namespace base_internal {
105
106// Like call_once, but uses KERNEL_ONLY scheduling. Intended to be used to
107// initialize entities used by the scheduler implementation.
108template <typename Callable, typename... Args>
109void LowLevelCallOnce(absl::Nonnull<absl::once_flag*> flag, Callable&& fn,
110 Args&&... args);
111
112// Disables scheduling while on stack when scheduling mode is non-cooperative.
113// No effect for cooperative scheduling modes.
114class SchedulingHelper {
115 public:
116 explicit SchedulingHelper(base_internal::SchedulingMode mode) : mode_(mode) {
117 if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) {
118 guard_result_ = base_internal::SchedulingGuard::DisableRescheduling();
119 }
120 }
121
122 ~SchedulingHelper() {
123 if (mode_ == base_internal::SCHEDULE_KERNEL_ONLY) {
124 base_internal::SchedulingGuard::EnableRescheduling(guard_result_);
125 }
126 }
127
128 private:
129 base_internal::SchedulingMode mode_;
130 bool guard_result_ = false;
131};
132
133// Bit patterns for call_once state machine values. Internal implementation
134// detail, not for use by clients.
135//
136// The bit patterns are arbitrarily chosen from unlikely values, to aid in
137// debugging. However, kOnceInit must be 0, so that a zero-initialized
138// once_flag will be valid for immediate use.
139enum {
140 kOnceInit = 0,
141 kOnceRunning = 0x65C2937B,
142 kOnceWaiter = 0x05A308D2,
143 // A very small constant is chosen for kOnceDone so that it fit in a single
144 // compare with immediate instruction for most common ISAs. This is verified
145 // for x86, POWER and ARM.
146 kOnceDone = 221, // Random Number
147};
148
149template <typename Callable, typename... Args>
150ABSL_ATTRIBUTE_NOINLINE void CallOnceImpl(
151 absl::Nonnull<std::atomic<uint32_t>*> control,
152 base_internal::SchedulingMode scheduling_mode, Callable&& fn,
153 Args&&... args) {
154#ifndef NDEBUG
155 {
156 uint32_t old_control = control->load(std::memory_order_relaxed);
157 if (old_control != kOnceInit &&
158 old_control != kOnceRunning &&
159 old_control != kOnceWaiter &&
160 old_control != kOnceDone) {
161 ABSL_RAW_LOG(FATAL, "Unexpected value for control word: 0x%lx",
162 static_cast<unsigned long>(old_control)); // NOLINT
163 }
164 }
165#endif // NDEBUG
166 static const base_internal::SpinLockWaitTransition trans[] = {
167 {.from: kOnceInit, .to: kOnceRunning, .done: true},
168 {.from: kOnceRunning, .to: kOnceWaiter, .done: false},
169 {.from: kOnceDone, .to: kOnceDone, .done: true}};
170
171 // Must do this before potentially modifying control word's state.
172 base_internal::SchedulingHelper maybe_disable_scheduling(scheduling_mode);
173 // Short circuit the simplest case to avoid procedure call overhead.
174 // The base_internal::SpinLockWait() call returns either kOnceInit or
175 // kOnceDone. If it returns kOnceDone, it must have loaded the control word
176 // with std::memory_order_acquire and seen a value of kOnceDone.
177 uint32_t old_control = kOnceInit;
178 if (control->compare_exchange_strong(i1&: old_control, i2: kOnceRunning,
179 m: std::memory_order_relaxed) ||
180 base_internal::SpinLockWait(w: control, ABSL_ARRAYSIZE(trans), trans,
181 scheduling_mode) == kOnceInit) {
182 base_internal::invoke(std::forward<Callable>(fn),
183 std::forward<Args>(args)...);
184 old_control =
185 control->exchange(i: base_internal::kOnceDone, m: std::memory_order_release);
186 if (old_control == base_internal::kOnceWaiter) {
187 base_internal::SpinLockWake(w: control, all: true);
188 }
189 } // else *control is already kOnceDone
190}
191
192inline absl::Nonnull<std::atomic<uint32_t>*> ControlWord(
193 absl::Nonnull<once_flag*> flag) {
194 return &flag->control_;
195}
196
197template <typename Callable, typename... Args>
198void LowLevelCallOnce(absl::Nonnull<absl::once_flag*> flag, Callable&& fn,
199 Args&&... args) {
200 std::atomic<uint32_t>* once = base_internal::ControlWord(flag);
201 uint32_t s = once->load(m: std::memory_order_acquire);
202 if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) {
203 base_internal::CallOnceImpl(once, base_internal::SCHEDULE_KERNEL_ONLY,
204 std::forward<Callable>(fn),
205 std::forward<Args>(args)...);
206 }
207}
208
209} // namespace base_internal
210
211template <typename Callable, typename... Args>
212void call_once(absl::once_flag& flag, Callable&& fn, Args&&... args) {
213 std::atomic<uint32_t>* once = base_internal::ControlWord(flag: &flag);
214 uint32_t s = once->load(m: std::memory_order_acquire);
215 if (ABSL_PREDICT_FALSE(s != base_internal::kOnceDone)) {
216 base_internal::CallOnceImpl(
217 once, base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL,
218 std::forward<Callable>(fn), std::forward<Args>(args)...);
219 }
220}
221
222ABSL_NAMESPACE_END
223} // namespace absl
224
225#endif // ABSL_BASE_CALL_ONCE_H_
226