1//
2// Copyright 2017 The Abseil Authors.
3//
4// Licensed under the Apache License, Version 2.0 (the "License");
5// you may not use this file except in compliance with the License.
6// You may obtain a copy of the License at
7//
8// https://www.apache.org/licenses/LICENSE-2.0
9//
10// Unless required by applicable law or agreed to in writing, software
11// distributed under the License is distributed on an "AS IS" BASIS,
12// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13// See the License for the specific language governing permissions and
14// limitations under the License.
15//
16// -----------------------------------------------------------------------------
17// File: optimization.h
18// -----------------------------------------------------------------------------
19//
20// This header file defines portable macros for performance optimization.
21//
22// This header is included in both C++ code and legacy C code and thus must
23// remain compatible with both C and C++. C compatibility will be removed if
24// the legacy code is removed or converted to C++. Do not include this header in
25// new code that requires C compatibility or assume C compatibility will remain
26// indefinitely.
27
28// SKIP_ABSL_INLINE_NAMESPACE_CHECK
29
30#ifndef ABSL_BASE_OPTIMIZATION_H_
31#define ABSL_BASE_OPTIMIZATION_H_
32
33#include <assert.h>
34
35#ifdef __cplusplus
36// Included for std::unreachable()
37#include <utility>
38#endif // __cplusplus
39
40#include "absl/base/config.h"
41#include "absl/base/options.h"
42
43// ABSL_BLOCK_TAIL_CALL_OPTIMIZATION
44//
45// Instructs the compiler to avoid optimizing tail-call recursion. This macro is
46// useful when you wish to preserve the existing function order within a stack
47// trace for logging, debugging, or profiling purposes.
48//
49// Example:
50//
51// int f() {
52// int result = g();
53// ABSL_BLOCK_TAIL_CALL_OPTIMIZATION();
54// return result;
55// }
56#if defined(__pnacl__)
57#define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; }
58#elif defined(__clang__)
59// Clang will not tail call given inline volatile assembly.
60#define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("")
61#elif defined(__GNUC__)
62// GCC will not tail call given inline volatile assembly.
63#define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() __asm__ __volatile__("")
64#elif defined(_MSC_VER)
65#include <intrin.h>
66// The __nop() intrinsic blocks the optimisation.
67#define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() __nop()
68#else
69#define ABSL_BLOCK_TAIL_CALL_OPTIMIZATION() if (volatile int x = 0) { (void)x; }
70#endif
71
72// ABSL_CACHELINE_SIZE
73//
74// Explicitly defines the size of the L1 cache for purposes of alignment.
75// Setting the cacheline size allows you to specify that certain objects be
76// aligned on a cacheline boundary with `ABSL_CACHELINE_ALIGNED` declarations.
77// (See below.)
78//
79// NOTE: this macro should be replaced with the following C++17 features, when
80// those are generally available:
81//
82// * `std::hardware_constructive_interference_size`
83// * `std::hardware_destructive_interference_size`
84//
85// See http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0154r1.html
86// for more information.
87#if defined(__GNUC__)
88// Cache line alignment
89#if defined(__i386__) || defined(__x86_64__)
90#define ABSL_CACHELINE_SIZE 64
91#elif defined(__powerpc64__)
92#define ABSL_CACHELINE_SIZE 128
93#elif defined(__aarch64__)
94// We would need to read special register ctr_el0 to find out L1 dcache size.
95// This value is a good estimate based on a real aarch64 machine.
96#define ABSL_CACHELINE_SIZE 64
97#elif defined(__arm__)
98// Cache line sizes for ARM: These values are not strictly correct since
99// cache line sizes depend on implementations, not architectures. There
100// are even implementations with cache line sizes configurable at boot
101// time.
102#if defined(__ARM_ARCH_5T__)
103#define ABSL_CACHELINE_SIZE 32
104#elif defined(__ARM_ARCH_7A__)
105#define ABSL_CACHELINE_SIZE 64
106#endif
107#endif
108#endif
109
110#ifndef ABSL_CACHELINE_SIZE
111// A reasonable default guess. Note that overestimates tend to waste more
112// space, while underestimates tend to waste more time.
113#define ABSL_CACHELINE_SIZE 64
114#endif
115
116// ABSL_CACHELINE_ALIGNED
117//
118// Indicates that the declared object be cache aligned using
119// `ABSL_CACHELINE_SIZE` (see above). Cacheline aligning objects allows you to
120// load a set of related objects in the L1 cache for performance improvements.
121// Cacheline aligning objects properly allows constructive memory sharing and
122// prevents destructive (or "false") memory sharing.
123//
124// NOTE: callers should replace uses of this macro with `alignas()` using
125// `std::hardware_constructive_interference_size` and/or
126// `std::hardware_destructive_interference_size` when C++17 becomes available to
127// them.
128//
129// See http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0154r1.html
130// for more information.
131//
132// On some compilers, `ABSL_CACHELINE_ALIGNED` expands to an `__attribute__`
133// or `__declspec` attribute. For compilers where this is not known to work,
134// the macro expands to nothing.
135//
136// No further guarantees are made here. The result of applying the macro
137// to variables and types is always implementation-defined.
138//
139// WARNING: It is easy to use this attribute incorrectly, even to the point
140// of causing bugs that are difficult to diagnose, crash, etc. It does not
141// of itself guarantee that objects are aligned to a cache line.
142//
143// NOTE: Some compilers are picky about the locations of annotations such as
144// this attribute, so prefer to put it at the beginning of your declaration.
145// For example,
146//
147// ABSL_CACHELINE_ALIGNED static Foo* foo = ...
148//
149// class ABSL_CACHELINE_ALIGNED Bar { ...
150//
151// Recommendations:
152//
153// 1) Consult compiler documentation; this comment is not kept in sync as
154// toolchains evolve.
155// 2) Verify your use has the intended effect. This often requires inspecting
156// the generated machine code.
157// 3) Prefer applying this attribute to individual variables. Avoid
158// applying it to types. This tends to localize the effect.
159#if defined(__clang__) || defined(__GNUC__)
160#define ABSL_CACHELINE_ALIGNED __attribute__((aligned(ABSL_CACHELINE_SIZE)))
161#elif defined(_MSC_VER)
162#define ABSL_CACHELINE_ALIGNED __declspec(align(ABSL_CACHELINE_SIZE))
163#else
164#define ABSL_CACHELINE_ALIGNED
165#endif
166
167// ABSL_PREDICT_TRUE, ABSL_PREDICT_FALSE
168//
169// Enables the compiler to prioritize compilation using static analysis for
170// likely paths within a boolean branch.
171//
172// Example:
173//
174// if (ABSL_PREDICT_TRUE(expression)) {
175// return result; // Faster if more likely
176// } else {
177// return 0;
178// }
179//
180// Compilers can use the information that a certain branch is not likely to be
181// taken (for instance, a CHECK failure) to optimize for the common case in
182// the absence of better information (ie. compiling gcc with `-fprofile-arcs`).
183//
184// Recommendation: Modern CPUs dynamically predict branch execution paths,
185// typically with accuracy greater than 97%. As a result, annotating every
186// branch in a codebase is likely counterproductive; however, annotating
187// specific branches that are both hot and consistently mispredicted is likely
188// to yield performance improvements.
189#if ABSL_HAVE_BUILTIN(__builtin_expect) || \
190 (defined(__GNUC__) && !defined(__clang__))
191#define ABSL_PREDICT_FALSE(x) (__builtin_expect(false || (x), false))
192#define ABSL_PREDICT_TRUE(x) (__builtin_expect(false || (x), true))
193#else
194#define ABSL_PREDICT_FALSE(x) (x)
195#define ABSL_PREDICT_TRUE(x) (x)
196#endif
197
198// `ABSL_INTERNAL_IMMEDIATE_ABORT_IMPL()` aborts the program in the fastest
199// possible way, with no attempt at logging. One use is to implement hardening
200// aborts with ABSL_OPTION_HARDENED. Since this is an internal symbol, it
201// should not be used directly outside of Abseil.
202#if ABSL_HAVE_BUILTIN(__builtin_trap) || \
203 (defined(__GNUC__) && !defined(__clang__))
204#define ABSL_INTERNAL_IMMEDIATE_ABORT_IMPL() __builtin_trap()
205#else
206#define ABSL_INTERNAL_IMMEDIATE_ABORT_IMPL() abort()
207#endif
208
209// `ABSL_INTERNAL_UNREACHABLE_IMPL()` is the platform specific directive to
210// indicate that a statement is unreachable, and to allow the compiler to
211// optimize accordingly. Clients should use `ABSL_UNREACHABLE()`, which is
212// defined below.
213#if defined(__cpp_lib_unreachable) && __cpp_lib_unreachable >= 202202L
214#define ABSL_INTERNAL_UNREACHABLE_IMPL() std::unreachable()
215#elif defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_unreachable)
216#define ABSL_INTERNAL_UNREACHABLE_IMPL() __builtin_unreachable()
217#elif ABSL_HAVE_BUILTIN(__builtin_assume)
218#define ABSL_INTERNAL_UNREACHABLE_IMPL() __builtin_assume(false)
219#elif defined(_MSC_VER)
220#define ABSL_INTERNAL_UNREACHABLE_IMPL() __assume(false)
221#else
222#define ABSL_INTERNAL_UNREACHABLE_IMPL()
223#endif
224
225// `ABSL_UNREACHABLE()` is an unreachable statement. A program which reaches
226// one has undefined behavior, and the compiler may optimize accordingly.
227#if ABSL_OPTION_HARDENED == 1 && defined(NDEBUG)
228// Abort in hardened mode to avoid dangerous undefined behavior.
229#define ABSL_UNREACHABLE() \
230 do { \
231 ABSL_INTERNAL_IMMEDIATE_ABORT_IMPL(); \
232 ABSL_INTERNAL_UNREACHABLE_IMPL(); \
233 } while (false)
234#else
235// The assert only fires in debug mode to aid in debugging.
236// When NDEBUG is defined, reaching ABSL_UNREACHABLE() is undefined behavior.
237#define ABSL_UNREACHABLE() \
238 do { \
239 /* NOLINTNEXTLINE: misc-static-assert */ \
240 assert(false && "ABSL_UNREACHABLE reached"); \
241 ABSL_INTERNAL_UNREACHABLE_IMPL(); \
242 } while (false)
243#endif
244
245// ABSL_ASSUME(cond)
246//
247// Informs the compiler that a condition is always true and that it can assume
248// it to be true for optimization purposes.
249//
250// WARNING: If the condition is false, the program can produce undefined and
251// potentially dangerous behavior.
252//
253// In !NDEBUG mode, the condition is checked with an assert().
254//
255// NOTE: The expression must not have side effects, as it may only be evaluated
256// in some compilation modes and not others. Some compilers may issue a warning
257// if the compiler cannot prove the expression has no side effects. For example,
258// the expression should not use a function call since the compiler cannot prove
259// that a function call does not have side effects.
260//
261// Example:
262//
263// int x = ...;
264// ABSL_ASSUME(x >= 0);
265// // The compiler can optimize the division to a simple right shift using the
266// // assumption specified above.
267// int y = x / 16;
268//
269#if !defined(NDEBUG)
270#define ABSL_ASSUME(cond) assert(cond)
271#elif ABSL_HAVE_BUILTIN(__builtin_assume)
272#define ABSL_ASSUME(cond) __builtin_assume(cond)
273#elif defined(_MSC_VER)
274#define ABSL_ASSUME(cond) __assume(cond)
275#elif defined(__cpp_lib_unreachable) && __cpp_lib_unreachable >= 202202L
276#define ABSL_ASSUME(cond) ((cond) ? void() : std::unreachable())
277#elif defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_unreachable)
278#define ABSL_ASSUME(cond) ((cond) ? void() : __builtin_unreachable())
279#elif ABSL_INTERNAL_CPLUSPLUS_LANG >= 202002L
280// Unimplemented. Uses the same definition as ABSL_ASSERT in the NDEBUG case.
281#define ABSL_ASSUME(expr) (decltype((expr) ? void() : void())())
282#else
283#define ABSL_ASSUME(expr) (false ? ((expr) ? void() : void()) : void())
284#endif
285
286// ABSL_INTERNAL_UNIQUE_SMALL_NAME(cond)
287// This macro forces small unique name on a static file level symbols like
288// static local variables or static functions. This is intended to be used in
289// macro definitions to optimize the cost of generated code. Do NOT use it on
290// symbols exported from translation unit since it may cause a link time
291// conflict.
292//
293// Example:
294//
295// #define MY_MACRO(txt)
296// namespace {
297// char VeryVeryLongVarName[] ABSL_INTERNAL_UNIQUE_SMALL_NAME() = txt;
298// const char* VeryVeryLongFuncName() ABSL_INTERNAL_UNIQUE_SMALL_NAME();
299// const char* VeryVeryLongFuncName() { return txt; }
300// }
301//
302
303#if defined(__GNUC__)
304#define ABSL_INTERNAL_UNIQUE_SMALL_NAME2(x) #x
305#define ABSL_INTERNAL_UNIQUE_SMALL_NAME1(x) ABSL_INTERNAL_UNIQUE_SMALL_NAME2(x)
306#define ABSL_INTERNAL_UNIQUE_SMALL_NAME() \
307 asm(ABSL_INTERNAL_UNIQUE_SMALL_NAME1(.absl.__COUNTER__))
308#else
309#define ABSL_INTERNAL_UNIQUE_SMALL_NAME()
310#endif
311
312#endif // ABSL_BASE_OPTIMIZATION_H_
313