Async++ unknown
Async (co_await/co_return) code for C++
Loading...
Searching...
No Matches
fiber.h
1#pragma once
2#include <cassert>
3#include <cstddef>
4#include <cstdint>
5#include <cstring>
6#include <exception>
7#include <memory>
8#include <optional>
9#include <stdexcept>
10#include <utility>
11
12#include <asyncpp/detail/sanitizers.h>
13#include <asyncpp/detail/std_import.h>
14
15#ifndef ASYNCPP_FIBER_KEYWORDS
16#define ASYNCPP_FIBER_KEYWORDS 1
17#endif
18
19#ifndef _WIN32
20
21#include <sys/mman.h>
22#include <unistd.h>
23
24#ifndef ASYNCPP_FIBER_USE_UCONTEXT
25#define ASYNCPP_FIBER_USE_UCONTEXT 0
26#endif
27
28#if ASYNCPP_FIBER_USE_UCONTEXT
29#include <ucontext.h>
30#endif
31
32#if !defined(MAP_ANON) && defined(__APPLE__)
33#define MAP_ANON 0x1000
34#endif
35
36namespace asyncpp::detail {
37 struct stack_context {
38 void* stack;
39 size_t stack_size;
40 void* mmap_base;
41 size_t mmap_size;
42 };
43
44 inline bool fiber_allocate_stack(stack_context& ctx, size_t size) noexcept {
45 static size_t pagesize = sysconf(_SC_PAGESIZE);
46
47 // Round the stacksize to the next multiple of pages
48 const auto page_count = (size + pagesize - 1) / pagesize;
49 size = page_count * pagesize;
50 const auto alloc_size = size + pagesize * 2;
51#if defined(MAP_STACK)
52 void* const stack =
53 ::mmap(nullptr, alloc_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON | MAP_STACK, -1, 0);
54#elif defined(MAP_ANON)
55 void* const stack = ::mmap(nullptr, alloc_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, -1, 0);
56#else
57 void* const stack = ::mmap(nullptr, alloc_size, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
58#endif
59 if (stack == MAP_FAILED) return false;
60
61 // Protect a page at the bottom and top to cause fault on stack over/underflow
62 [[maybe_unused]] auto res = ::mprotect(stack, pagesize, PROT_NONE);
63 assert(res == 0);
64 res = ::mprotect(static_cast<std::byte*>(stack) + size + pagesize, pagesize, PROT_NONE);
65 assert(res == 0);
66
67 ctx.stack = static_cast<std::byte*>(stack) + size + pagesize;
68 ctx.stack_size = size;
69 ctx.mmap_base = stack;
70 ctx.mmap_size = alloc_size;
71 return true;
72 }
73
74 inline bool fiber_deallocate_stack(stack_context& ctx) {
75 if (ctx.mmap_base == nullptr) return true;
76
77 auto res = ::munmap(ctx.mmap_base, ctx.mmap_size);
78 return res == 0;
79 }
80
81#if ASYNCPP_FIBER_USE_UCONTEXT
82 struct fiber_context {
83 ucontext_t context;
84 void* stack;
85 size_t stack_size;
86 };
87
88 inline bool fiber_makecontext(fiber_context* ctx, const stack_context& stack, void (*fn)(void* arg), void* arg) {
89 static void (*const wrapper)(uint32_t, uint32_t, uint32_t, uint32_t) = [](uint32_t fn_hi, uint32_t fn_low,
90 uint32_t arg_hi, uint32_t arg_low) {
91 uintptr_t fn = (static_cast<uintptr_t>(fn_hi) << 32) | static_cast<uintptr_t>(fn_low);
92 uintptr_t arg = (static_cast<uintptr_t>(arg_hi) << 32) | static_cast<uintptr_t>(arg_low);
93 reinterpret_cast<void (*)(void*)>(fn)(reinterpret_cast<void*>(arg));
94 };
95 getcontext(&ctx->context);
96 ctx->context.uc_link = nullptr;
97 ctx->context.uc_stack.ss_sp = reinterpret_cast<decltype(ctx->context.uc_stack.ss_sp)>(
98 reinterpret_cast<uintptr_t>(stack.stack) - stack.stack_size);
99 ctx->context.uc_stack.ss_size = stack.stack_size;
100 makecontext(&ctx->context, reinterpret_cast<void (*)()>(wrapper), 4,
101 (reinterpret_cast<uintptr_t>(fn) >> 32) & 0xffffffff, reinterpret_cast<uintptr_t>(fn) & 0xffffffff,
102 (reinterpret_cast<uintptr_t>(arg) >> 32) & 0xffffffff,
103 reinterpret_cast<uintptr_t>(arg) & 0xffffffff);
104 return true;
105 }
106
107 inline bool fiber_swapcontext(fiber_context* out, fiber_context* in) {
108 swapcontext(&out->context, &in->context);
109 return true;
110 }
111
112 inline bool fiber_destroy_context(fiber_context* ctx) {
113 memset(ctx, 0, sizeof(fiber_context));
114 return true;
115 }
116#else
117 struct fiber_context {
118 void* current_sp;
119 void* stack;
120 size_t stack_size;
121 };
122
123 inline bool fiber_makecontext(fiber_context* ctx, const stack_context& stack, void (*entry_fn)(void* arg),
124 void* arg) {
125 ctx->stack = stack.stack;
126 ctx->stack_size = stack.stack_size;
127
128 // Make sure top of the stack is aligned to 16byte. This should always be the case but better safe than sorry.
129 // NOLINTNEXTLINE(performance-no-int-to-ptr,readability-identifier-length)
130 auto sp = reinterpret_cast<uintptr_t*>(reinterpret_cast<uintptr_t>(stack.stack) & static_cast<uintptr_t>(~15L));
131#if defined(__i386__)
132 // The restore code adds 28 byte and ret another 4. Since we want
133 // ($esp & 0xf) == 0xc (as would be the case after a call),
134 // we need to make sure this is already the case.
135 sp -= 3;
136 struct stack_frame {
137 uint32_t mmx_cw;
138 uint32_t x87_cw;
139 uintptr_t stack_guard;
140 uintptr_t edi;
141 uintptr_t esi;
142 uintptr_t ebx;
143 uintptr_t ebp;
144 uintptr_t ret_addr;
145 // This would normally be the return address of the
146 // called function if it was called via `call`.
147 // However because we abuse a ret to jump to the entry
148 // its not pushed.
149 uintptr_t : sizeof(uintptr_t) * 8;
150 uintptr_t param;
151 };
152 static_assert(sizeof(stack_frame) == sizeof(uintptr_t) * 10);
153 sp -= sizeof(stack_frame) / sizeof(uintptr_t);
154 memset(sp, 0, reinterpret_cast<uintptr_t>(stack.stack) - reinterpret_cast<uintptr_t>(sp));
155 auto frame = reinterpret_cast<stack_frame*>(sp);
156 frame->ret_addr = reinterpret_cast<uintptr_t>(entry_fn);
157 frame->param = reinterpret_cast<uintptr_t>(arg);
158 assert((reinterpret_cast<uintptr_t>(sp) % 16) == 0xc);
159#elif defined(__x86_64__)
160 // The restore code adds 72 byte and ret another 8. Since we want
161 // ($esp & 0xf) == 0x8 (as would be the case after a call),
162 // we need to make sure this is already the case.
163 sp -= 1;
164 struct stack_frame {
165 uint32_t mmx_cw;
166 uint32_t x87_cw;
167 uintptr_t stack_guard;
168 uintptr_t r12;
169 uintptr_t r13;
170 uintptr_t r14;
171 uintptr_t r15;
172 uintptr_t rbx;
173 uintptr_t rbp;
174 union {
175 uintptr_t rdi;
176 uintptr_t param;
177 };
178 uintptr_t ret_addr;
179 };
180 static_assert(sizeof(stack_frame) == sizeof(uintptr_t) * 10);
181 sp -= sizeof(stack_frame) / sizeof(uintptr_t);
182 memset(sp, 0, reinterpret_cast<uintptr_t>(stack.stack) - reinterpret_cast<uintptr_t>(sp));
183 auto frame = reinterpret_cast<stack_frame*>(sp);
184 frame->ret_addr = reinterpret_cast<uintptr_t>(entry_fn);
185 frame->param = reinterpret_cast<uintptr_t>(arg);
186#elif defined(__arm__)
187 // Stack needs to be 64bit aligned for vstmia
188 sp -= 16;
189 struct stack_frame {
190#if (defined(__VFP_FP__) && !defined(__SOFTFP__))
191 uintptr_t d8_d15[16];
192 uintptr_t _padding_;
193#endif
194 union {
195 uintptr_t r0;
196 uintptr_t param;
197 };
198 uintptr_t r11;
199 uintptr_t r10;
200 uintptr_t r9;
201 uintptr_t r8;
202 uintptr_t r7;
203 uintptr_t r6;
204 uintptr_t r5;
205 uintptr_t r4;
206 uintptr_t r12;
207 union {
208 uintptr_t r14;
209 uintptr_t ret_addr;
210 };
211 };
212#if (defined(__VFP_FP__) && !defined(__SOFTFP__))
213 static_assert(sizeof(stack_frame) == sizeof(uintptr_t) * 28);
214#else
215 static_assert(sizeof(stack_frame) == sizeof(uintptr_t) * 11);
216#endif
217 sp -= sizeof(stack_frame) / sizeof(uintptr_t);
218 memset(sp, 0, reinterpret_cast<uintptr_t>(stack.stack) - reinterpret_cast<uintptr_t>(sp));
219 auto frame = reinterpret_cast<stack_frame*>(sp);
220 frame->ret_addr = reinterpret_cast<uintptr_t>(entry_fn);
221 frame->param = reinterpret_cast<uintptr_t>(arg);
222#elif defined(__arm64__) || defined(__aarch64__)
223 // ARM64 requires the stack pointer to always be 16 byte aligned
224 sp -= 16;
225 struct stack_frame {
226 uintptr_t d8_d15[8];
227 uintptr_t x19_x29[11];
228 union {
229 uintptr_t x30;
230 uintptr_t ret_addr;
231 };
232 union {
233 uintptr_t x0;
234 uintptr_t param;
235 };
236 uintptr_t _padding_;
237 };
238 static_assert(sizeof(stack_frame) == sizeof(uintptr_t) * 22);
239 sp -= sizeof(stack_frame) / sizeof(uintptr_t);
240 memset(sp, 0, reinterpret_cast<uintptr_t>(stack.stack) - reinterpret_cast<uintptr_t>(sp));
241 auto frame = reinterpret_cast<stack_frame*>(sp);
242 frame->ret_addr = reinterpret_cast<uintptr_t>(entry_fn);
243 frame->param = reinterpret_cast<uintptr_t>(arg);
244#elif defined(__s390x__)
245 // The s390x ABI requires a 160-byte caller allocated register save area.
246 sp -= 20;
247 struct stack_frame {
248 uintptr_t f8_15[8];
249 uintptr_t r6_13[8];
250 union {
251 uintptr_t r14;
252 uintptr_t ret_addr;
253 };
254 union {
255 uintptr_t r2;
256 uintptr_t param;
257 };
258 };
259 static_assert(sizeof(stack_frame) == sizeof(uintptr_t) * 18);
260 sp -= sizeof(stack_frame) / sizeof(uintptr_t);
261 memset(sp, 0, reinterpret_cast<uintptr_t>(stack.stack) - reinterpret_cast<uintptr_t>(sp));
262 auto frame = reinterpret_cast<stack_frame*>(sp);
263 frame->ret_addr = reinterpret_cast<uintptr_t>(entry_fn);
264 frame->param = reinterpret_cast<uintptr_t>(arg);
265#elif defined(__powerpc64__)
266 sp -= 4;
267 struct stack_frame {
268 uintptr_t r2;
269 union {
270 uintptr_t r3;
271 uintptr_t param;
272 };
273 uintptr_t r12;
274 uintptr_t r14_31[18];
275 uintptr_t fr14_31[18];
276 uintptr_t cr;
277 uintptr_t lr;
278 uintptr_t ret_addr;
279 };
280 static_assert(sizeof(stack_frame) == sizeof(uintptr_t) * 42);
281 sp -= sizeof(stack_frame) / sizeof(uintptr_t);
282 memset(sp, 0, reinterpret_cast<uintptr_t>(stack.stack) - reinterpret_cast<uintptr_t>(sp));
283 auto frame = reinterpret_cast<stack_frame*>(sp);
284 // Note: On PPC a function has two entry points for local and global entry respectively
285 // Since we get the global entry point we also need to set the r12 register to the
286 // same pointer.
287 frame->r12 = reinterpret_cast<uintptr_t>(entry_fn);
288 frame->ret_addr = reinterpret_cast<uintptr_t>(entry_fn);
289 frame->param = reinterpret_cast<uintptr_t>(arg);
290#else
291#error "Unsupported architecture."
292#endif
293 ctx->current_sp = sp;
294 return true;
295 }
296
297 __attribute__((naked, noinline)) inline void fiber_swap_stack(void** current_pointer_out, void* dest_pointer) {
298#if defined(__i386__)
299 /* `current_pointer_out` is in `4(%ebp)`. `dest_pointer` is in `8(%ebp)`. */
300 //NOLINTNEXTLINE(hicpp-no-assembler)
301 asm("leal -0x1c(%esp), %esp\n"
302 // Save FPU state
303 "stmxcsr (%esp)\n" // save MMX control- and status-word
304 "fnstcw 0x4(%esp)\n" // save x87 control-word
305#ifdef SAVE_TLS_STACK_PROTECTOR
306 "movl %gs:0x14, %ecx\n" // read stack guard from TLS record
307 "movl %ecx, 0x8(%esp)\n" // save stack guard
308#endif
309 "movl %edi, 0xc(%esp)\n" // save EDI
310 "movl %esi, 0x10(%esp)\n" // save ESI
311 "movl %ebx, 0x14(%esp)\n" // save EBX
312 "movl %ebp, 0x18(%esp)\n" // save EBP
313
314 "movl 0x20(%esp), %ecx\n" // Read first arg ...
315 "movl %esp, (%ecx)\n" // ... and copy previous stack pointer there
316
317 "mov 0x24(%esp), %esi\n" // Read second arg ...
318 "mov %esi, %esp\n" // ... and restore it to the stack pointer
319
320 "ldmxcsr (%esp)\n" // restore MMX control- and status-word
321 "fldcw 0x4(%esp)\n" // restore x87 control-word
322#ifdef SAVE_TLS_STACK_PROTECTOR
323 "movl 0x8(%esp), %edx\n" // load stack guard
324 "movl %edx, %gs:0x14\n" // restore stack guard to TLS record
325#endif
326 "movl 0xc(%esp), %edi\n" // restore EDI
327 "movl 0x10(%esp), %esi\n" // restore ESI
328 "movl 0x14(%esp), %ebx\n" // restore EBX
329 "movl 0x18(%esp), %ebp\n" // restore EBP
330
331 "leal 0x1c(%esp), %esp\n" // Adjust stack $esp => &ret_addr
332
333 "ret\n"); // ip = *$esp; $esp -= 4; => $esp = &_padding_;
334#elif defined(__x86_64__)
335#if !(defined(__LP64__) || defined(__LLP64__))
336// Having non-native-sized pointers makes things very messy.
337#error "Non-native pointer size."
338#endif
339 /* `current_pointer_out` is in `%rdi`. `dest_pointer` is in `%rsi`. */
340 //NOLINTNEXTLINE(hicpp-no-assembler)
341 asm("leaq -0x48(%rsp), %rsp\n"
342 // Save FPU state
343 "stmxcsr (%rsp)\n" /* save MMX control- and status-word */
344 "fnstcw 0x4(%rsp)\n" /* save x87 control-word */
345#ifdef SAVE_TLS_STACK_PROTECTOR
346 "movq %fs:0x28, %rcx\n" /* read stack guard from TLS record */
347 "movq %rcx, 0x8(%rsp)\n" /* save stack guard */
348#endif
349 "movq %r12, 0x10(%rsp)\n" // save R12
350 "movq %r13, 0x18(%rsp)\n" // save R13
351 "movq %r14, 0x20(%rsp)\n" // save R14
352 "movq %r15, 0x28(%rsp)\n" // save R15
353 "movq %rbx, 0x30(%rsp)\n" // save RBX
354 "movq %rbp, 0x38(%rsp)\n" // save RBP
355 // On amd64, the first argument comes from rdi.
356 "movq %rsp, (%rdi)\n"
357 // On amd64, the second argument comes from rsi.
358 "movq %rsi, %rsp\n"
359 // Restore FPU state
360 "ldmxcsr (%rsp)\n" // Restore MMX control- and status-word
361 "fldcw 0x4(%rsp)\n" // Restore x87 control-word
362#ifdef SAVE_TLS_STACK_PROTECTOR
363 "movq 0x8(%rsp), %rcx\n" // Restore stack guard to TLS record
364 "movq %rcx, %fs:28\n"
365#endif
366 // Restore callee saved registers
367 "movq 0x10(%rsp), %r12\n"
368 "movq 0x18(%rsp), %r13\n"
369 "movq 0x20(%rsp), %r14\n"
370 "movq 0x28(%rsp), %r15\n"
371 "movq 0x30(%rsp), %rbx\n"
372 "movq 0x38(%rsp), %rbp\n"
373 "movq 0x40(%rsp), %rdi\n"
374 // Adjust stack ptr
375 "leaq 0x48(%rsp), %rsp\n"
376 // Return to the restored function
377 "ret\n");
378#elif defined(__arm__)
379 /* `current_pointer_out` is in `r0`. `dest_pointer` is in `r1` */
380 //NOLINTNEXTLINE(hicpp-no-assembler)
381 asm(
382 // Stack is 64bit aligned by the caller, preserve that.
383 // Preserve r4-r12 and the return address (r14).
384 "push {r12,r14}\n"
385 "push {r4-r11}\n"
386#if (defined(__VFP_FP__) && !defined(__SOFTFP__))
387 "sub sp, sp, #72\n"
388 "vstmia sp, {d8-d15}\n"
389#else
390 "sub sp, sp, #8\n"
391#endif
392 // On ARM, the first argument is in `r0`, the second argument is in `r1` and `r13` is the stack pointer.
393 "str r13, [r0]\n"
394 "mov r13, r1\n"
395 // Restore callee save registers
396#if (defined(__VFP_FP__) && !defined(__SOFTFP__))
397 "vldmia sp, {d8-d15}\n"
398 "add sp, sp, #68\n"
399#else
400 "sub sp, sp, #4\n"
401#endif
402 "pop {r0}\n"
403 "pop {r4-r11}\n"
404 "pop {r12, r14}\n"
405 // Return to the restored function
406 "bx r14\n");
407#elif defined(__arm64__) || defined(__aarch64__)
408 //NOLINTNEXTLINE(hicpp-no-assembler)
409 asm(
410 // Preserve d8-d15 + x19-x29 and the return address (x30).
411 // Note: x30 is stored twice due to alignment requirements
412 "sub sp, sp, #0xb0\n"
413 "stp d8, d9, [sp, #0x00]\n"
414 "stp d10, d11, [sp, #0x10]\n"
415 "stp d12, d13, [sp, #0x20]\n"
416 "stp d14, d15, [sp, #0x30]\n"
417 "stp x19, x20, [sp, #0x40]\n"
418 "stp x21, x22, [sp, #0x50]\n"
419 "stp x23, x24, [sp, #0x60]\n"
420 "stp x25, x26, [sp, #0x70]\n"
421 "stp x27, x28, [sp, #0x80]\n"
422 "stp x29, x30, [sp, #0x90]\n"
423 // On ARM64, the first argument is in `x0`. the second argument is in `x1`,
424 // `sp` is the stack pointer and `x4` is a scratch register. */
425 "mov x4, sp\n"
426 "str x4, [x0]\n"
427 "mov sp, x1\n"
428 // Restore callee saved registers
429 "ldp d8, d9, [sp, #0x00]\n"
430 "ldp d10, d11, [sp, #0x10]\n"
431 "ldp d12, d13, [sp, #0x20]\n"
432 "ldp d14, d15, [sp, #0x30]\n"
433 "ldp x19, x20, [sp, #0x40]\n"
434 "ldp x21, x22, [sp, #0x50]\n"
435 "ldp x23, x24, [sp, #0x60]\n"
436 "ldp x25, x26, [sp, #0x70]\n"
437 "ldp x27, x28, [sp, #0x80]\n"
438 "ldp x29, x30, [sp, #0x90]\n"
439 "ldr x0, [sp, #0xa0]\n"
440 "add sp, sp, #0xb0\n"
441 // Return to the restored function
442 "ret x30\n");
443#elif defined(__s390x__)
444 /* `current_pointer_out` is in `%r2`. `dest_pointer` is in `%r3`. */
445 //NOLINTNEXTLINE(hicpp-no-assembler)
446 asm(
447 // Preserve r6-r13, the return address (r14), and f8-f15.
448 "aghi %r15, -(8*18)\n"
449
450 "stmg %r6, %r14, 64(%r15)\n"
451 "std %f8, (8*0)(%r15)\n"
452 "std %f9, (8*1)(%r15)\n"
453 "std %f10, (8*2)(%r15)\n"
454 "std %f11, (8*3)(%r15)\n"
455 "std %f12, (8*4)(%r15)\n"
456 "std %f13, (8*5)(%r15)\n"
457 "std %f14, (8*6)(%r15)\n"
458 "std %f15, (8*7)(%r15)\n"
459 // On s390x, the first argument is in r2. r15 is the stack pointer.
460 "stg %r15, 0(%r2)\n"
461 // On s390x, the second argument is in r3
462 "lgr %r15, %r3\n"
463 // Restore callee saved registers
464 "lmg %r6, %r14, (8*8)(%r15)\n"
465 "ld %f8, (8*0)(%r15)\n"
466 "ld %f9, (8*1)(%r15)\n"
467 "ld %f10, (8*2)(%r15)\n"
468 "ld %f11, (8*3)(%r15)\n"
469 "ld %f12, (8*4)(%r15)\n"
470 "ld %f13, (8*5)(%r15)\n"
471 "ld %f14, (8*6)(%r15)\n"
472 "ld %f15, (8*7)(%r15)\n"
473 "lg %r2, (8*17)(%r15)\n"
474
475 "aghi %r15, (8*18)\n"
476 // Return to the restored function
477 "br %r14\n");
478#elif defined(__powerpc64__)
479 // `current_pointer_out` is in `r3`. `dest_pointer` is in `r4`
480 //NOLINTNEXTLINE(hicpp-no-assembler)
481 asm("addis %r2, %r12, .TOC.-fiber_swap_stack@ha\n"
482 "addi %r2, %r2, .TOC.-fiber_swap_stack@l\n"
483 ".localentry fiber_swap_stack, . - fiber_swap_stack\n"
484 "addi %r1, %r1, -(42*8)\n"
485 // Note: technically we only need to persist r2 and r3 on elfv1,
486 // but we do it always to stay consistent
487 "std %r2, (8*0)(%r1)\n"
488 "std %r3, (8*1)(%r1)\n"
489 "std %r12, (8*2)(%r1)\n"
490 "std %r14, (8*3)(%r1)\n"
491 "std %r15, (8*4)(%r1)\n"
492 "std %r16, (8*5)(%r1)\n"
493 "std %r17, (8*6)(%r1)\n"
494 "std %r18, (8*7)(%r1)\n"
495 "std %r19, (8*8)(%r1)\n"
496 "std %r20, (8*9)(%r1)\n"
497 "std %r21, (8*10)(%r1)\n"
498 "std %r22, (8*11)(%r1)\n"
499 "std %r23, (8*12)(%r1)\n"
500 "std %r24, (8*13)(%r1)\n"
501 "std %r25, (8*14)(%r1)\n"
502 "std %r26, (8*15)(%r1)\n"
503 "std %r27, (8*16)(%r1)\n"
504 "std %r28, (8*17)(%r1)\n"
505 "std %r29, (8*18)(%r1)\n"
506 "std %r30, (8*19)(%r1)\n"
507 "std %r31, (8*20)(%r1)\n"
508 "stfd %f14, (8*21)(%r1)\n"
509 "stfd %f15, (8*22)(%r1)\n"
510 "stfd %f16, (8*23)(%r1)\n"
511 "stfd %f17, (8*24)(%r1)\n"
512 "stfd %f18, (8*25)(%r1)\n"
513 "stfd %f19, (8*26)(%r1)\n"
514 "stfd %f20, (8*27)(%r1)\n"
515 "stfd %f21, (8*28)(%r1)\n"
516 "stfd %f22, (8*29)(%r1)\n"
517 "stfd %f23, (8*30)(%r1)\n"
518 "stfd %f24, (8*31)(%r1)\n"
519 "stfd %f25, (8*32)(%r1)\n"
520 "stfd %f26, (8*33)(%r1)\n"
521 "stfd %f27, (8*34)(%r1)\n"
522 "stfd %f28, (8*35)(%r1)\n"
523 "stfd %f29, (8*36)(%r1)\n"
524 "stfd %f30, (8*37)(%r1)\n"
525 "stfd %f31, (8*38)(%r1)\n"
526 "mfcr %r0\n"
527 "std %r0, (8*39)(%r1)\n"
528 "mflr %r0\n"
529 "std %r0, (8*40)(%r1)\n"
530 "std %r0, (8*41)(%r1)\n"
531 // Save old stack pointer.
532 "std %r1, 0(%r3)\n"
533 // Load the new stack pointer
534 "mr %r1, %r4\n"
535 // Load preserved registers
536 "ld %r2, (8*0)(%r1)\n"
537 "ld %r3, (8*1)(%r1)\n"
538 "ld %r12, (8*2)(%r1)\n"
539 "ld %r14, (8*3)(%r1)\n"
540 "ld %r15, (8*4)(%r1)\n"
541 "ld %r16, (8*5)(%r1)\n"
542 "ld %r17, (8*6)(%r1)\n"
543 "ld %r18, (8*7)(%r1)\n"
544 "ld %r19, (8*8)(%r1)\n"
545 "ld %r20, (8*9)(%r1)\n"
546 "ld %r21, (8*10)(%r1)\n"
547 "ld %r22, (8*11)(%r1)\n"
548 "ld %r23, (8*12)(%r1)\n"
549 "ld %r24, (8*13)(%r1)\n"
550 "ld %r25, (8*14)(%r1)\n"
551 "ld %r26, (8*15)(%r1)\n"
552 "ld %r27, (8*16)(%r1)\n"
553 "ld %r28, (8*17)(%r1)\n"
554 "ld %r29, (8*18)(%r1)\n"
555 "ld %r30, (8*19)(%r1)\n"
556 "ld %r31, (8*20)(%r1)\n"
557 "lfd %f14,(8*21)(%r1)\n"
558 "lfd %f15,(8*22)(%r1)\n"
559 "lfd %f16,(8*23)(%r1)\n"
560 "lfd %f17,(8*24)(%r1)\n"
561 "lfd %f18,(8*25)(%r1)\n"
562 "lfd %f19,(8*26)(%r1)\n"
563 "lfd %f20,(8*27)(%r1)\n"
564 "lfd %f21,(8*28)(%r1)\n"
565 "lfd %f22,(8*29)(%r1)\n"
566 "lfd %f23,(8*30)(%r1)\n"
567 "lfd %f24,(8*31)(%r1)\n"
568 "lfd %f25,(8*32)(%r1)\n"
569 "lfd %f26,(8*33)(%r1)\n"
570 "lfd %f27,(8*34)(%r1)\n"
571 "lfd %f28,(8*35)(%r1)\n"
572 "lfd %f29,(8*36)(%r1)\n"
573 "lfd %f30,(8*37)(%r1)\n"
574 "lfd %f31,(8*38)(%r1)\n"
575 "ld %r0, (8*39)(%r1)\n"
576 "mtcr %r0\n"
577 "ld %r0, (8*40)(%r1)\n"
578 "mtlr %r0\n"
579 "ld %r0, (8*41)(%r1)\n"
580 "mtctr %r0\n"
581 "addi %r1, %r1, (8*42)\n"
582 // Return to restored function
583 "bctr\n");
584#else
585#error "Unsupported architecture."
586#endif
587 }
588
589 inline bool fiber_swapcontext(fiber_context* out_ctx, fiber_context* in_ctx) {
590 fiber_swap_stack(&out_ctx->current_sp, in_ctx->current_sp);
591 return true;
592 }
593
594 inline bool fiber_destroy_context(fiber_context* ctx) {
595 memset(ctx, 0, sizeof(fiber_context));
596 return true;
597 }
598#endif
599} // namespace asyncpp::detail
600#else
601
602#include <windows.h>
603
604namespace asyncpp::detail {
605 struct stack_context {
606 // WinFiber manages the stack itself, so we only need the size
607 size_t stack_size;
608 };
609
610 inline bool fiber_allocate_stack(stack_context& ctx, size_t size) noexcept {
611 static size_t pagesize = []() {
612 SYSTEM_INFO si;
613 ::GetSystemInfo(&si);
614 return static_cast<size_t>(si.dwPageSize);
615 }();
616
617 // Round the stacksize to the next multiple of pages
618 const auto page_count = (size + pagesize - 1) / pagesize;
619 size = page_count * pagesize;
620 ctx.stack_size = size;
621 return true;
622 }
623
624 inline bool fiber_deallocate_stack(stack_context& ctx) noexcept { return true; }
625
626 struct fiber_context {
627 LPVOID fiber_handle;
628 void (*start_fn)(void* arg);
629 void* start_arg;
630 };
631
632 inline bool fiber_makecontext(fiber_context* ctx, const stack_context& stack, void (*entry_fn)(void* arg),
633 void* arg) {
634 static void (*wrapper)(LPVOID) = [](LPVOID param) {
635 auto ctx = static_cast<fiber_context*>(param);
636 ctx->start_fn(ctx->start_arg);
637 };
638 ctx->fiber_handle = CreateFiber(stack.stack_size, wrapper, ctx);
639 if (ctx->fiber_handle == NULL) return false;
640 ctx->start_fn = entry_fn;
641 ctx->start_arg = arg;
642 return true;
643 }
644
645 inline bool fiber_swapcontext(fiber_context* out, fiber_context* in) {
646#if (_WIN32_WINNT > 0x0600)
647 if (::IsThreadAFiber() == FALSE) {
648 out->fiber_handle = ::ConvertThreadToFiber(nullptr);
649 } else {
650 out->fiber_handle = ::GetCurrentFiber();
651 }
652#else
653 out->fiber_handle = ::ConvertThreadToFiber(nullptr);
654 if (out->fiber_handle == NULL) {
655 if (::GetLastError() != ERROR_ALREADY_FIBER) return false;
656 out->fiber_handle = ::GetCurrentFiber();
657 }
658#endif
659 if (out->fiber_handle == NULL) return false;
660 SwitchToFiber(in->fiber_handle);
661 return true;
662 }
663
664 inline bool fiber_destroy_context(fiber_context* ctx) {
665 DeleteFiber(ctx->fiber_handle);
666 ctx->fiber_handle = NULL;
667 return true;
668 }
669
670} // namespace asyncpp::detail
671#endif
672namespace asyncpp {
673 template<typename TReturn>
674 class fiber;
675} // namespace asyncpp
676namespace asyncpp::detail {
677 struct fiber_handle_base {
678 // C++20 coroutine ABI dictates those
679 void (*resume_cb)(fiber_handle_base*) = nullptr;
680 void (*destroy_cb)(fiber_handle_base*) = nullptr;
681
682 detail::fiber_context main_context{};
683 detail::fiber_context fiber_context{};
684 detail::stack_context fiber_stack{};
685
686 bool (*suspend_handler)(void* ptr, coroutine_handle<> hndl) = nullptr;
687 void* suspend_handler_ptr = nullptr;
688 std::exception_ptr suspend_exception = nullptr;
689
690 coroutine_handle<> continuation{};
691
692 bool want_destroy = false;
693 bool was_started = false;
694#if ASYNCPP_HAS_ASAN
695 void* asan_handle = nullptr;
696 const void* asan_parent_stack = nullptr;
697 size_t asan_parent_stack_size{};
698#endif
699#if ASYNCPP_HAS_TSAN
700 void* tsan_parent = nullptr;
701 void* tsan_fiber = nullptr;
702#endif
703#if ASYNCPP_HAS_VALGRIND
704 unsigned valgrind_id = 0;
705#endif
706 };
707
708#if defined(ASYNCPP_SO_COMPAT)
709 extern thread_local fiber_handle_base* g_current_fiber;
710#else
711 inline static thread_local fiber_handle_base* g_current_fiber = nullptr;
712#endif
713
714#if defined(ASYNCPP_SO_COMPAT_IMPL)
715 thread_local fiber_handle_base* g_current_fiber = nullptr;
716#endif
717
718 struct fiber_destroy_requested_exception {};
719
720 template<typename FN>
721 static coroutine_handle<> make_fiber_handle(size_t stack_size, FN&& cbfn) {
722 struct handle : fiber_handle_base {
723 FN function;
724 explicit handle(FN&& cbfn) : function(std::forward<FN>(cbfn)) {}
725 };
726 static_assert(offsetof(handle, resume_cb) == 0);
727
728 auto hndl = new handle(std::forward<FN>(cbfn));
729 hndl->resume_cb = [](fiber_handle_base* hndl) {
730 auto old = std::exchange(g_current_fiber, hndl);
731 while (hndl->resume_cb != nullptr) {
732#if ASYNCPP_HAS_ASAN
733 void* asan_handle;
734 __sanitizer_start_switch_fiber(&asan_handle, hndl->fiber_stack.mmap_base, hndl->fiber_stack.mmap_size);
735#endif
736#if ASYNCPP_HAS_TSAN
737 hndl->tsan_parent = __tsan_get_current_fiber();
738 __tsan_switch_to_fiber(hndl->tsan_fiber, 0);
739#endif
740 detail::fiber_swapcontext(&hndl->main_context, &hndl->fiber_context);
741#if ASYNCPP_HAS_ASAN
742 const void* fiber_stack;
743 size_t fiber_stack_size;
744 __sanitizer_finish_switch_fiber(asan_handle, &fiber_stack, &fiber_stack_size);
745 assert(fiber_stack == hndl->fiber_stack.mmap_base);
746 assert(fiber_stack_size == hndl->fiber_stack.mmap_size);
747#endif
748 if (hndl->suspend_handler) {
749 auto handler = std::exchange(hndl->suspend_handler, nullptr);
750 auto ptr = std::exchange(hndl->suspend_handler_ptr, nullptr);
751 try {
752 if (handler(ptr, coroutine_handle<>::from_address(static_cast<void*>(hndl)))) break;
753 } catch (...) { hndl->suspend_exception = std::current_exception(); }
754 }
755 }
756 g_current_fiber = old;
757 if (hndl->resume_cb == nullptr && hndl->continuation != nullptr) { hndl->continuation.resume(); }
758 };
759 hndl->destroy_cb = [](fiber_handle_base* hndl) {
760 // Signal destruction and resume, then delete self
761 if (hndl->resume_cb != nullptr && hndl->was_started) {
762 hndl->want_destroy = true;
763 hndl->resume_cb(hndl);
764 assert(hndl->resume_cb == nullptr);
765 }
766#if ASYNCPP_HAS_TSAN
767 __tsan_destroy_fiber(hndl->tsan_fiber);
768#endif
769#if ASYNCPP_HAS_VALGRIND
770 VALGRIND_STACK_DEREGISTER(hndl->valgrind_id);
771#endif
772 detail::fiber_destroy_context(&hndl->fiber_context);
773 detail::fiber_deallocate_stack(hndl->fiber_stack);
774 delete static_cast<handle*>(hndl);
775 };
776#if ASYNCPP_HAS_TSAN
777 hndl->tsan_fiber = __tsan_create_fiber(0);
778#endif
779 if (!detail::fiber_allocate_stack(hndl->fiber_stack, stack_size)) {
780#if ASYNCPP_HAS_TSAN
781 __tsan_destroy_fiber(hndl->tsan_fiber);
782#endif
783 delete hndl;
784 throw std::bad_alloc();
785 }
786 if (!detail::fiber_makecontext(
787 &hndl->fiber_context, hndl->fiber_stack,
788 [](void* ptr) {
789 auto hndl = static_cast<handle*>(ptr);
790#if ASYNCPP_HAS_ASAN
791 __sanitizer_finish_switch_fiber(hndl->asan_handle, &hndl->asan_parent_stack,
792 &hndl->asan_parent_stack_size);
793#endif
794 hndl->was_started = true;
795 try {
796 hndl->function();
797 } catch (const fiber_destroy_requested_exception&) {} // NOLINT(bugprone-empty-catch)
798 hndl->resume_cb = nullptr;
799#if ASYNCPP_HAS_ASAN
800 __sanitizer_start_switch_fiber(nullptr, hndl->asan_parent_stack, hndl->asan_parent_stack_size);
801#endif
802#if ASYNCPP_HAS_TSAN
803 assert(hndl->tsan_fiber == __tsan_get_current_fiber());
804 __tsan_switch_to_fiber(hndl->tsan_parent, 0);
805#endif
806 detail::fiber_swapcontext(&hndl->fiber_context, &hndl->main_context);
807 assert(false);
808 },
809 hndl)) {
810#if ASYNCPP_HAS_TSAN
811 __tsan_destroy_fiber(hndl->tsan_fiber);
812#endif
813 fiber_deallocate_stack(hndl->fiber_stack);
814 delete hndl;
815 throw std::bad_alloc();
816 }
817#if ASYNCPP_HAS_VALGRIND
818 hndl->valgrind_id =
819 VALGRIND_STACK_REGISTER(hndl->fiber_stack.mmap_base,
820 static_cast<uint8_t*>(hndl->fiber_stack.mmap_base) + hndl->fiber_stack.mmap_size);
821#endif
822 return coroutine_handle<>::from_address(static_cast<void*>(hndl));
823 }
824
825 template<typename T>
826 static auto fiber_await(T&& awaiter) {
827 // We can not suspend during exception unwinding because the unwinding library
828 // uses thread local globals.
829 if (std::uncaught_exceptions() != 0) std::terminate();
830 if (!static_cast<bool>(awaiter.await_ready())) {
831 // Awaiting is a coordination between this fiber handle's
832 // resume method and the fiber itself. Because the fiber is
833 // expected to be fully suspened when await_suspend is called
834 // we can't directly do so here. Rather we build a callback to
835 // invoke awaiter.await_suspend with the correct parameters and
836 // switch_context to the resume() call. There we evaluate
837 // await_suspend, passing the handle, and resume the correct handle
838 // if needed. Special handling is needed for exceptions thrown in
839 // await_suspend (which need to get rethrown in the fiber).
840 auto hndl = detail::g_current_fiber;
841 assert(hndl != nullptr);
842#ifdef __linux__
843 assert(__builtin_frame_address(0) > hndl->fiber_stack.mmap_base);
844 assert(__builtin_frame_address(0) <
845 (static_cast<uint8_t*>(hndl->fiber_stack.mmap_base) + hndl->fiber_stack.mmap_size));
846#endif
847 using AwaitResult = decltype(awaiter.await_suspend(std::declval<coroutine_handle<>>()));
848 hndl->suspend_handler = [](void* ptr, coroutine_handle<> hndl) {
849 if constexpr (std::is_same_v<AwaitResult, void>) {
850 static_cast<T*>(ptr)->await_suspend(hndl);
851 return true;
852 } else if constexpr (std::is_same_v<AwaitResult, bool>) {
853 return static_cast<T*>(ptr)->await_suspend(hndl);
854 } else { // Treat everything else as a coroutine_handle to resume
855 static_cast<T*>(ptr)->await_suspend(hndl).resume();
856 return true;
857 }
858 };
859 hndl->suspend_handler_ptr = &awaiter;
860#if ASYNCPP_HAS_ASAN
861 void* asan_handle;
862 __sanitizer_start_switch_fiber(&asan_handle, hndl->asan_parent_stack, hndl->asan_parent_stack_size);
863#endif
864#if ASYNCPP_HAS_TSAN
865 assert(hndl->tsan_fiber == __tsan_get_current_fiber());
866 __tsan_switch_to_fiber(hndl->tsan_parent, 0);
867#endif
868 detail::fiber_swapcontext(&hndl->fiber_context, &hndl->main_context);
869#if ASYNCPP_HAS_ASAN
870 __sanitizer_finish_switch_fiber(asan_handle, &hndl->asan_parent_stack, &hndl->asan_parent_stack_size);
871#endif
872 if (hndl->suspend_exception != nullptr)
873 std::rethrow_exception(std::exchange(hndl->suspend_exception, nullptr));
874 if (hndl->want_destroy) {
875 // NOLINTNEXTLINE(hicpp-exception-baseclass)
876 throw fiber_destroy_requested_exception{};
877 }
878 }
879 return awaiter.await_resume();
880 }
881
882 struct fib_await_helper {
883 template<typename T>
884 // NOLINTNEXTLINE(misc-unconventional-assign-operator)
885 auto operator=(T&& awaiter) {
886 return fiber_await(std::forward<T>(awaiter));
887 }
888 };
889} // namespace asyncpp::detail
890
891namespace asyncpp {
895 template<>
896 class fiber<void> {
897
898 coroutine_handle<> m_handle;
899
900 public:
913 template<typename FN>
914 explicit fiber(FN&& function, size_t stack_size = 262144)
915 requires(!std::is_same_v<FN, fiber>)
916 : m_handle(detail::make_fiber_handle(stack_size, std::forward<FN>(function))) {}
918 constexpr fiber() noexcept : m_handle(nullptr) {}
920 ~fiber() {
921 if (m_handle) m_handle.destroy();
922 }
924 fiber(fiber&& other) noexcept : m_handle(std::exchange(other.m_handle, nullptr)) {}
926 fiber& operator=(fiber&& other) noexcept {
927 if (&other != this) {
928 if (m_handle) m_handle.destroy();
929 m_handle = std::exchange(other.m_handle, nullptr);
930 }
931 return *this;
932 }
933 fiber(const fiber&) = delete;
934 fiber& operator=(const fiber&) = delete;
935
943 template<typename T>
944 static auto fiber_await(T&& awaiter) {
945 return detail::fiber_await(std::forward<decltype(awaiter)>(awaiter));
946 }
947
952 auto await() {
953 if (!m_handle) throw std::logic_error("empty fiber");
954 struct awaiter {
955 coroutine_handle<> m_handle;
956 [[nodiscard]] bool await_ready() const noexcept { return m_handle.done(); }
957 [[nodiscard]] coroutine_handle<> await_suspend(coroutine_handle<> hdl) const {
958 auto ctx = static_cast<detail::fiber_handle_base*>(m_handle.address());
959 if (ctx->continuation != nullptr) throw std::logic_error("already awaited");
960 ctx->continuation = hdl;
961 return m_handle;
962 }
963 constexpr void await_resume() const noexcept {}
964 };
965 return awaiter{m_handle};
966 }
967
972 auto operator co_await() { return await(); }
973 };
974
978 template<typename TReturn>
979 class fiber {
980 std::unique_ptr<std::optional<TReturn>> m_result{};
981 fiber<void> m_base{};
982
983 public:
996 template<typename FN>
997 explicit fiber(FN&& function, size_t stack_size = 262144)
998 requires(!std::is_same_v<FN, fiber>)
999 : m_result(std::make_unique<std::optional<TReturn>>()),
1000 m_base([function = std::forward<FN>(function), res = m_result.get()]() { res->emplace(function()); },
1001 stack_size) {}
1003 constexpr fiber() noexcept = default;
1005 fiber(fiber&& other) noexcept : m_result(std::move(other.m_result)), m_base(std::move(other.m_base)) {}
1007 fiber& operator=(fiber&& other) noexcept {
1008 if (&other != this) {
1009 m_base = std::move(other.m_base);
1010 m_result = std::move(other.m_result);
1011 }
1012 return *this;
1013 }
1014 fiber(const fiber&) = delete;
1015 fiber& operator=(const fiber&) = delete;
1016
1024 template<typename T>
1025 static auto fiber_await(T&& awaiter) {
1026 return detail::fiber_await(std::forward<T>(awaiter));
1027 }
1028
1033 auto await() & {
1034 if (!m_result) throw std::logic_error("empty fiber");
1035 struct awaiter {
1036 decltype(m_base.operator co_await()) m_awaiter;
1037 std::optional<TReturn>* m_result;
1038 [[nodiscard]] bool await_ready() const noexcept { return m_awaiter.await_ready(); }
1039 [[nodiscard]] coroutine_handle<> await_suspend(coroutine_handle<> hdl) const {
1040 return m_awaiter.await_suspend(hdl);
1041 }
1042 auto await_resume() const noexcept {
1043 assert(m_result->has_value());
1044 return m_result->value();
1045 }
1046 };
1047 return awaiter{m_base.await(), m_result.get()};
1048 }
1049
1054 auto await() && {
1055 if (!m_result) throw std::logic_error("empty fiber");
1056 struct awaiter {
1057 decltype(m_base.operator co_await()) m_awaiter;
1058 std::optional<TReturn>* m_result;
1059 [[nodiscard]] bool await_ready() const noexcept { return m_awaiter.await_ready(); }
1060 [[nodiscard]] coroutine_handle<> await_suspend(coroutine_handle<> hdl) const {
1061 return m_awaiter.await_suspend(hdl);
1062 }
1063 auto await_resume() const noexcept {
1064 assert(m_result->has_value());
1065 return std::move(m_result->value());
1066 }
1067 };
1068 return awaiter{m_base.await(), m_result.get()};
1069 }
1070
1075 auto operator co_await() & { return await(); }
1080 auto operator co_await() && { return await(); }
1081 };
1082
1083 template<typename FN>
1084 fiber(FN&&) -> fiber<std::invoke_result_t<FN>>;
1085} // namespace asyncpp
1086
1087#if ASYNCPP_FIBER_KEYWORDS
1088#define fib_await (asyncpp::detail::fib_await_helper{}) =
1089#endif
asyncpp::fiber< void >::~fiber
~fiber()
Destructor.
Definition fiber.h:920
asyncpp::fiber< void >::await
auto await()
Get an awaitable for this fiber.
Definition fiber.h:952
asyncpp::fiber< void >::fiber
constexpr fiber() noexcept
Construct an empty fiber handle.
Definition fiber.h:918
asyncpp::fiber< void >::operator=
fiber & operator=(fiber &&other) noexcept
Move constructor. The moved from handle will be empty.
Definition fiber.h:926
asyncpp::fiber< void >::fiber
fiber(FN &&function, size_t stack_size=262144)
Construct a new fiber for the specified entry function.
Definition fiber.h:914
asyncpp::fiber< void >::fiber_await
static auto fiber_await(T &&awaiter)
Await a standard C++20 coroutine awaitable.
Definition fiber.h:944
asyncpp::fiber< void >::fiber
fiber(fiber &&other) noexcept
Move constructor. The moved from handle will be empty.
Definition fiber.h:924
asyncpp::fiber
Fiber class providing a stackfull coroutine.
Definition fiber.h:979
asyncpp::fiber::fiber_await
static auto fiber_await(T &&awaiter)
Await a standard C++20 coroutine awaitable.
Definition fiber.h:1025
asyncpp::fiber::fiber
constexpr fiber() noexcept=default
Construct an empty fiber handle.
asyncpp::fiber::await
auto await() &&
Get an awaitable for this fiber.
Definition fiber.h:1054
asyncpp::fiber::operator=
fiber & operator=(fiber &&other) noexcept
Move constructor. The moved from handle will be empty.
Definition fiber.h:1007
asyncpp::fiber::await
auto await() &
Get an awaitable for this fiber.
Definition fiber.h:1033
asyncpp::fiber::fiber
fiber(FN &&function, size_t stack_size=262144)
Construct a new fiber for the specified entry function.
Definition fiber.h:997
std_import.h
Provides a consistent import interface for coroutine, experimental/coroutine or a best effort fallbac...