1//
2// Copyright 2017 The Abseil Authors.
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8// https://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15//
16// -----------------------------------------------------------------------------
17// File: optimization.h
18// -----------------------------------------------------------------------------
19//
20// This header file defines portable macros for performance optimization.
21
22#ifndef ABSL_BASE_OPTIMIZATION_H_
23#define ABSL_BASE_OPTIMIZATION_H_
24
25#include <assert.h>
26
27#include "absl/base/config.h"
28#include "absl/base/options.h"
29
30// ABSL_BLOCK_TAIL_CALL_OPTIMIZATION
31//
32// Instructs the compiler to avoid optimizing tail-call recursion. This macro is
33// useful when you wish to preserve the existing function order within a stack
34// trace for logging, debugging, or profiling purposes.
35//
36// Example:
37//
38// int f() {
39// int result = g();
40// ABSL_BLOCK_TAIL_CALL_OPTIMIZATION();
41// return result;
42// }
43#if defined(__pnacl__)
44#define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; }
45#elif defined(__clang__)
46// Clang will not tail call given inline volatile assembly.
47#define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("")
48#elif defined(__GNUC__)
49// GCC will not tail call given inline volatile assembly.
50#define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("")
51#elif defined(_MSC_VER)
52#include <intrin.h>
53// The __nop() intrinsic blocks the optimisation.
54#define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() __nop()
55#else
56#define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; }
57#endif
58
59// ABSL_CACHELINE_SIZE
60//
61// Explicitly defines the size of the L1 cache for purposes of alignment.
62// Setting the cacheline size allows you to specify that certain objects be
63// aligned on a cacheline boundary with `ABSL_CACHELINE_ALIGNED` declarations.
64// (See below.)
65//
66// NOTE: this macro should be replaced with the following C++17 features, when
67// those are generally available:
68//
69// * `std::hardware_constructive_interference_size`
70// * `std::hardware_destructive_interference_size`
71//
72// See http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0154r1.html
73// for more information.
74#if defined(__GNUC__)
75// Cache line alignment
76#if defined(__i386__) || defined(__x86_64__)
77#define ABSL_CACHELINE_SIZE 64
78#elif defined(__powerpc64__)
79#define ABSL_CACHELINE_SIZE 128
80#elif defined(__aarch64__)
81// We would need to read special register ctr_el0 to find out L1 dcache size.
82// This value is a good estimate based on a real aarch64 machine.
83#define ABSL_CACHELINE_SIZE 64
84#elif defined(__arm__)
85// Cache line sizes for ARM: These values are not strictly correct since
86// cache line sizes depend on implementations, not architectures. There
87// are even implementations with cache line sizes configurable at boot
88// time.
89#if defined(__ARM_ARCH_5T__)
90#define ABSL_CACHELINE_SIZE 32
91#elif defined(__ARM_ARCH_7A__)
92#define ABSL_CACHELINE_SIZE 64
93#endif
94#endif
95#endif
96
97#ifndef ABSL_CACHELINE_SIZE
98// A reasonable default guess. Note that overestimates tend to waste more
99// space, while underestimates tend to waste more time.
100#define ABSL_CACHELINE_SIZE 64
101#endif
102
103// ABSL_CACHELINE_ALIGNED
104//
105// Indicates that the declared object be cache aligned using
106// `ABSL_CACHELINE_SIZE` (see above). Cacheline aligning objects allows you to
107// load a set of related objects in the L1 cache for performance improvements.
108// Cacheline aligning objects properly allows constructive memory sharing and
109// prevents destructive (or "false") memory sharing.
110//
111// NOTE: callers should replace uses of this macro with `alignas()` using
112// `std::hardware_constructive_interference_size` and/or
113// `std::hardware_destructive_interference_size` when C++17 becomes available to
114// them.
115//
116// See http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0154r1.html
117// for more information.
118//
119// On some compilers, `ABSL_CACHELINE_ALIGNED` expands to an `__attribute__`
120// or `__declspec` attribute. For compilers where this is not known to work,
121// the macro expands to nothing.
122//
123// No further guarantees are made here. The result of applying the macro
124// to variables and types is always implementation-defined.
125//
126// WARNING: It is easy to use this attribute incorrectly, even to the point
127// of causing bugs that are difficult to diagnose, crash, etc. It does not
128// of itself guarantee that objects are aligned to a cache line.
129//
130// NOTE: Some compilers are picky about the locations of annotations such as
131// this attribute, so prefer to put it at the beginning of your declaration.
132// For example,
133//
134// ABSL_CACHELINE_ALIGNED static Foo* foo = ...
135//
136// class ABSL_CACHELINE_ALIGNED Bar { ...
137//
138// Recommendations:
139//
140// 1) Consult compiler documentation; this comment is not kept in sync as
141// toolchains evolve.
142// 2) Verify your use has the intended effect. This often requires inspecting
143// the generated machine code.
144// 3) Prefer applying this attribute to individual variables. Avoid
145// applying it to types. This tends to localize the effect.
146#if defined(__clang__) || defined(__GNUC__)
147#define ABSL_CACHELINE_ALIGNED __attribute__((aligned(ABSL_CACHELINE_SIZE)))
148#elif defined(_MSC_VER)
149#define ABSL_CACHELINE_ALIGNED __declspec(align(ABSL_CACHELINE_SIZE))
150#else
151#define ABSL_CACHELINE_ALIGNED
152#endif
153
154// ABSL_PREDICT_TRUE, ABSL_PREDICT_FALSE
155//
156// Enables the compiler to prioritize compilation using static analysis for
157// likely paths within a boolean branch.
158//
159// Example:
160//
161// if (ABSL_PREDICT_TRUE(expression)) {
162// return result; // Faster if more likely
163// } else {
164// return 0;
165// }
166//
167// Compilers can use the information that a certain branch is not likely to be
168// taken (for instance, a CHECK failure) to optimize for the common case in
169// the absence of better information (ie. compiling gcc with `-fprofile-arcs`).
170//
171// Recommendation: Modern CPUs dynamically predict branch execution paths,
172// typically with accuracy greater than 97%. As a result, annotating every
173// branch in a codebase is likely counterproductive; however, annotating
174// specific branches that are both hot and consistently mispredicted is likely
175// to yield performance improvements.
176#if ABSL_HAVE_BUILTIN(__builtin_expect) || \
177 (defined(__GNUC__) && !defined(__clang__))
178#define ABSL_PREDICT_FALSE(x) (__builtin_expect(false || (x), false))
179#define ABSL_PREDICT_TRUE(x) (__builtin_expect(false || (x), true))
180#else
181#define ABSL_PREDICT_FALSE(x) (x)
182#define ABSL_PREDICT_TRUE(x) (x)
183#endif
184
185// `ABSL_INTERNAL_IMMEDIATE_ABORT_IMPL()` aborts the program in the fastest
186// possible way, with no attempt at logging. One use is to implement hardening
187// aborts with ABSL_OPTION_HARDENED. Since this is an internal symbol, it
188// should not be used directly outside of Abseil.
189#if ABSL_HAVE_BUILTIN(__builtin_trap) || \
190 (defined(__GNUC__) && !defined(__clang__))
191#define ABSL_INTERNAL_IMMEDIATE_ABORT_IMPL() __builtin_trap()
192#else
193#define ABSL_INTERNAL_IMMEDIATE_ABORT_IMPL() abort()
194#endif
195
196// `ABSL_INTERNAL_UNREACHABLE_IMPL()` is the platform specific directive to
197// indicate that a statement is unreachable, and to allow the compiler to
198// optimize accordingly. Clients should use `ABSL_UNREACHABLE()`, which is
199// defined below.
200#if defined(__cpp_lib_unreachable) && __cpp_lib_unreachable >= 202202L
201#define ABSL_INTERNAL_UNREACHABLE_IMPL() std::unreachable()
202#elif defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_unreachable)
203#define ABSL_INTERNAL_UNREACHABLE_IMPL() __builtin_unreachable()
204#elif ABSL_HAVE_BUILTIN(__builtin_assume)
205#define ABSL_INTERNAL_UNREACHABLE_IMPL() __builtin_assume(false)
206#elif defined(_MSC_VER)
207#define ABSL_INTERNAL_UNREACHABLE_IMPL() __assume(false)
208#else
209#define ABSL_INTERNAL_UNREACHABLE_IMPL()
210#endif
211
212// `ABSL_UNREACHABLE()` is an unreachable statement. A program which reaches
213// one has undefined behavior, and the compiler may optimize accordingly.
214#if ABSL_OPTION_HARDENED == 1 && defined(NDEBUG)
215// Abort in hardened mode to avoid dangerous undefined behavior.
216#define ABSL_UNREACHABLE() \
217 do { \
218 ABSL_INTERNAL_IMMEDIATE_ABORT_IMPL(); \
219 ABSL_INTERNAL_UNREACHABLE_IMPL(); \
220 } while (false)
221#else
222// The assert only fires in debug mode to aid in debugging.
223// When NDEBUG is defined, reaching ABSL_UNREACHABLE() is undefined behavior.
224#define ABSL_UNREACHABLE() \
225 do { \
226 /* NOLINTNEXTLINE: misc-static-assert */ \
227 assert(false && "ABSL_UNREACHABLE reached"); \
228 ABSL_INTERNAL_UNREACHABLE_IMPL(); \
229 } while (false)
230#endif
231
232// ABSL_ASSUME(cond)
233//
234// Informs the compiler that a condition is always true and that it can assume
235// it to be true for optimization purposes.
236//
237// WARNING: If the condition is false, the program can produce undefined and
238// potentially dangerous behavior.
239//
240// In !NDEBUG mode, the condition is checked with an assert().
241//
242// NOTE: The expression must not have side effects, as it may only be evaluated
243// in some compilation modes and not others. Some compilers may issue a warning
244// if the compiler cannot prove the expression has no side effects. For example,
245// the expression should not use a function call since the compiler cannot prove
246// that a function call does not have side effects.
247//
248// Example:
249//
250// int x = ...;
251// ABSL_ASSUME(x >= 0);
252// // The compiler can optimize the division to a simple right shift using the
253// // assumption specified above.
254// int y = x / 16;
255//
256#if !defined(NDEBUG)
257#define ABSL_ASSUME(cond) assert(cond)
258#elif ABSL_HAVE_BUILTIN(__builtin_assume)
259#define ABSL_ASSUME(cond) __builtin_assume(cond)
260#elif defined(_MSC_VER)
261#define ABSL_ASSUME(cond) __assume(cond)
262#elif defined(__cpp_lib_unreachable) && __cpp_lib_unreachable >= 202202L
263#define ABSL_ASSUME(cond) \
264 do { \
265 if (!(cond)) std::unreachable(); \
266 } while (false)
267#elif defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_unreachable)
268#define ABSL_ASSUME(cond) \
269 do { \
270 if (!(cond)) __builtin_unreachable(); \
271 } while (false)
272#else
273#define ABSL_ASSUME(cond) \
274 do { \
275 static_cast<void>(false && (cond)); \
276 } while (false)
277#endif
278
279// ABSL_INTERNAL_UNIQUE_SMALL_NAME(cond)
280// This macro forces small unique name on a static file level symbols like
281// static local variables or static functions. This is intended to be used in
282// macro definitions to optimize the cost of generated code. Do NOT use it on
283// symbols exported from translation unit since it may cause a link time
284// conflict.
285//
286// Example:
287//
288// #define MY_MACRO(txt)
289// namespace {
290// char VeryVeryLongVarName[] ABSL_INTERNAL_UNIQUE_SMALL_NAME() = txt;
291// const char* VeryVeryLongFuncName() ABSL_INTERNAL_UNIQUE_SMALL_NAME();
292// const char* VeryVeryLongFuncName() { return txt; }
293// }
294//
295
296#if defined(__GNUC__)
297#define ABSL_INTERNAL_UNIQUE_SMALL_NAME2(x) #x
298#define ABSL_INTERNAL_UNIQUE_SMALL_NAME1(x) ABSL_INTERNAL_UNIQUE_SMALL_NAME2(x)
299#define ABSL_INTERNAL_UNIQUE_SMALL_NAME() \
300 asm(ABSL_INTERNAL_UNIQUE_SMALL_NAME1(.absl.__COUNTER__))
301#else
302#define ABSL_INTERNAL_UNIQUE_SMALL_NAME()
303#endif
304
305#endif // ABSL_BASE_OPTIMIZATION_H_
306