async_task_promise.hpp

Go to the documentation of this file.

 
#pragma once
 
#include "async_errors.hpp"
#include "async_node.hpp"
#include "async_task.hpp"
#include "scheduler.hpp"
#include <atomic>
#include <coroutine>
#include <exception>
#include <limits>
#include <memory>
#include <optional>
#include <type_traits>
#include <vector>
 
namespace uni20::async
{
 
class TaskFactory;
class EpochContext;
 
struct CudaTaskAwaiterTag
{};
 
template <typename T>
concept CudaTaskAwaitable = std::derived_from<std::remove_cvref_t<T>, CudaTaskAwaiterTag>;
 
void propagate_unhandled_writer_exception(EpochContext* epoch, std::exception_ptr eptr) noexcept;
 
template <typename Ret>
concept AwaitSuspendResult = std::same_as<Ret, void> || std::same_as<Ret, BasicTask>;
 
template <typename T>
concept TaskAwaitable = requires(T a, BasicTask task) {
  { a.await_suspend(std::move(task)) } noexcept -> AwaitSuspendResult;
};
 
template <typename T>
concept TaskFactoryChildAwaitable = requires(T a, BasicTask task) {
  { a.await_suspend(std::move(task)) } noexcept -> std::same_as<void>;
};
 
template <typename T>
concept TaskFactoryAwaitable = requires(T a, TaskFactory factory) {
  { a.await_suspend(std::move(factory)) } noexcept -> std::same_as<void>;
  { a.num_awaiters() } -> std::convertible_to<int>;
};
 
template <TaskAwaitable A> struct TaskAwaiter;
 
template <TaskFactoryAwaitable A> struct TaskFactoryAwaiter;
 
class TaskPromiseBase {
  public:
    struct ExceptionSinkNode
    {
        TaskPromiseBase* owner{nullptr};
        ExceptionSinkNode* prev{nullptr};
        ExceptionSinkNode* next{nullptr};
        std::shared_ptr<EpochContext> epoch{};
        bool explicit_sink{false};
    };
 
    std::atomic<bool> started_{false};
 
    TaskHandle continuation_{};
 
    std::exception_ptr* continuation_exception_ = nullptr;
 
    std::atomic<int> awaiter_count_ = 0;
 
    std::atomic<bool> exception_{false};
    std::exception_ptr eptr_ = nullptr;
 
    std::atomic<bool> cancel_on_resume_{false};
 
    static constexpr int kNoPreferredNumaNode = std::numeric_limits<int>::min();
 
    std::atomic<int> preferred_numa_node_{kNoPreferredNumaNode};
 
    // debugging / DAG info
    std::string Name;  // function name of the coroutine
    uint64_t Instance; // instance number, global
    ExceptionSinkNode* exception_sinks_head_{nullptr};
 
#if UNI20_DEBUG_DAG
    // For debug tracking the DAG, we store the nodes of the incoming (ReadBuffer) and outgoing (WriteBuffer) objects.
    std::vector<NodeInfo const*> ReadDependencies;
    std::vector<NodeInfo const*> WriteDependencies;
#endif
 
    // /// \brief To propogate exceptions and cancellations to the appropriate awaiter, whenever we suspend
    // /// we stash the awaiter here, so we can pass on set_cancel() and set_exception()
    // AsyncAwaiter* current_awaiter_ = nullptr;
    //
    // void set_current_awaiter(AsyncAwaiter* a) { current_awaiter_ = a; }
    //
    // void set_cancel() noexcept { cancel_.store(true, std::memory_order_release); }
    //
    // void set_exception(std::exception_ptr e) noexcept
    // {
    //   eptr_ = e;
    //   // write to the atomic, mostly for the memory barrier
    //   awaiter_has_error_.store(false, std::memory_order_release);
    // }
    //
    // bool is_cancel_on_resume() const noexcept { return awaiter_has_error_.load(std::memory_order_acquire); }
    //
    // std::exception_ptr is_exception_on_resume() const noexcept
    // {
    //   return awaiter_has_error_.load(std::memory_order_acquire) ? nullptr : eptr_;
    // }
 
    template <typename... Args> TaskPromiseBase(Args&&... args)
    {
      // For each parameter, detect ReadBuffer / WriteBuffer
      (ProcessCoroutineArgument(this, args), ...);
    }
 
    TaskHandle destroy_with_continuation() noexcept
    {
      auto c = continuation_;
      continuation_ = {};
      continuation_exception_ = nullptr;
      auto handle = this->self();
      TaskPromiseBase::destroy_and_track(handle);
      return c;
    }
 
    ~TaskPromiseBase() noexcept
    {
      DEBUG_CHECK(!continuation_);
      DEBUG_CHECK(continuation_exception_ == nullptr);
    }
 
    bool release_awaiter() noexcept { return awaiter_count_.fetch_sub(1, std::memory_order_acq_rel) == 1; }
 
    bool release_awaiter(int count) noexcept
    {
      return awaiter_count_.fetch_sub(count, std::memory_order_acq_rel) == count;
    }
 
    int add_awaiter() noexcept { return awaiter_count_.fetch_add(1, std::memory_order_relaxed); }
 
    int add_awaiter(int count) noexcept { return awaiter_count_.fetch_add(count, std::memory_order_relaxed); }
 
    void set_exception(std::exception_ptr e) noexcept
    {
      if (!exception_.exchange(true, std::memory_order_acq_rel))
      {
        eptr_ = e;
      }
    }
 
    void register_exception_sink(ExceptionSinkNode& node, std::shared_ptr<EpochContext> epoch, bool explicit_sink)
    {
      if (node.owner) node.owner->unregister_exception_sink(node, false);
      if (!epoch) return;
 
      node.owner = this;
      node.epoch = std::move(epoch);
      node.explicit_sink = explicit_sink;
      node.prev = nullptr;
      node.next = exception_sinks_head_;
      if (exception_sinks_head_) exception_sinks_head_->prev = &node;
      exception_sinks_head_ = &node;
    }
 
    void unregister_exception_sink(ExceptionSinkNode& node, bool from_destructor) noexcept
    {
      if (node.owner != this) return;
      if (from_destructor && node.explicit_sink && std::uncaught_exceptions() > 0)
      {
        CHECK(
            false,
            "propagate_exceptions_to sink destroyed during exception unwinding before coroutine unhandled_exception()");
      }
 
      if (node.prev)
        node.prev->next = node.next;
      else
        exception_sinks_head_ = node.next;
 
      if (node.next) node.next->prev = node.prev;
 
      node.owner = nullptr;
      node.prev = nullptr;
      node.next = nullptr;
      node.epoch.reset();
      node.explicit_sink = false;
    }
 
    std::exception_ptr get_exception() noexcept
    {
      if (exception_.load(std::memory_order_acquire))
        return eptr_;
      else
        return nullptr;
    }
 
    void rethrow_exception()
    {
      if (exception_.load(std::memory_order_acquire)) std::rethrow_exception(eptr_);
    }
 
    void set_cancel_on_resume() noexcept { cancel_on_resume_.store(true, std::memory_order_release); }
 
    [[nodiscard]] bool is_cancel_on_resume() const noexcept
    {
      return cancel_on_resume_.load(std::memory_order_acquire);
    }
 
    template <TaskAwaitable A> auto await_transform(A& a);
    template <TaskAwaitable A> auto await_transform(A&& a);
 
    template <TaskFactoryAwaitable A> auto await_transform(A& a);
    template <TaskFactoryAwaitable A> auto await_transform(A&& a);
 
    template <typename Task>
      requires std::derived_from<std::remove_cvref_t<Task>, BasicTask>
    Task&& await_transform(Task&& task) noexcept
    {
      return std::forward<Task>(task);
    }
 
    template <typename T> auto await_transform(T&&)
    {
      static_assert(sizeof(T) == 0,
                    "co_await expression does not match any known TaskAwaitable or TaskFactoryAwaitable type.");
    }
 
    void set_preferred_numa_node(std::optional<int> node) noexcept
    {
      preferred_numa_node_.store(node ? *node : kNoPreferredNumaNode, std::memory_order_release);
    }
 
    [[nodiscard]] std::optional<int> preferred_numa_node() const noexcept
    {
      int node = preferred_numa_node_.load(std::memory_order_acquire);
      if (node == kNoPreferredNumaNode) return std::nullopt;
      return node;
    }
 
    void mark_started() noexcept { started_.store(true, std::memory_order_release); }
 
    [[nodiscard]] bool has_started() const noexcept { return started_.load(std::memory_order_acquire); }
 
    [[nodiscard]] IScheduler* scheduler() const noexcept { return sched_; }
 
    [[nodiscard]] TaskHandle self() noexcept
    {
      DEBUG_PRECONDITION(self_);
      return self_;
    }
 
    static void note_running(TaskHandle handle) noexcept { TaskRegistry::mark_running(handle.coroutine()); }
 
    static void note_suspended(TaskHandle handle) noexcept { TaskRegistry::mark_suspended(handle.coroutine()); }
 
#if UNI20_DEBUG_DAG
    template <typename A> static void note_await_dependency(TaskHandle handle, A const& awaitable)
    {
      auto record = [coroutine = handle.coroutine()](NodeInfo const* node, TaskRegistry::EpochTaskRole role) {
        TaskRegistry::record_await_dependency(coroutine, node, role);
      };
      if constexpr (requires { awaitable.debug_each_dependency(record); })
      {
        awaitable.debug_each_dependency(record);
      }
      else if constexpr (requires {
                           awaitable.node();
                           awaitable.debug_task_role();
                         })
      {
        TaskRegistry::record_await_dependency(handle.coroutine(), awaitable.node(), awaitable.debug_task_role());
      }
    }
#endif
 
    static void note_leaked(TaskHandle handle) noexcept { TaskRegistry::leak_task(handle.coroutine()); }
 
    static void resume_and_track(TaskHandle handle)
    {
      note_running(handle);
      handle.coroutine().resume();
    }
 
    static void destroy_and_track(TaskHandle handle) noexcept
    {
      TaskRegistry::destroy_task(handle.coroutine());
      handle.coroutine().destroy();
    }
 
    static void prepare_nested_route(TaskHandle parent, TaskHandle child);
 
    [[nodiscard]] static bool can_transfer_directly(TaskHandle from, TaskHandle to) noexcept;
 
    static std::coroutine_handle<> resolve_await_suspend_result(TaskHandle current, BasicTask&& task)
    {
      if (!task)
      {
        note_suspended(current);
        return std::noop_coroutine();
      }
 
      auto nested = task.release_ownership();
      CHECK(nested, "coroutine handle was not exclusively owned!");
 
      if (nested == current)
      {
        note_running(current);
        return current.coroutine();
      }
 
      prepare_nested_route(current, nested);
      nested.promise().continuation_ = current;
      note_suspended(current);
 
      if (!can_transfer_directly(current, nested))
      {
        auto nested_task = nested.promise().take_ownership();
        BasicTask::reschedule(std::move(nested_task));
        return std::noop_coroutine();
      }
 
      nested.promise().mark_started();
      note_running(nested);
      return nested.coroutine();
    }
 
    template <TaskAwaitable A> static auto suspend_task_awaitable(TaskHandle current, A& awaitable);
 
    template <TaskFactoryAwaitable A> static void suspend_factory_awaitable(TaskHandle current, A& awaitable);
 
    BasicTask take_ownership() noexcept
    {
      [[maybe_unused]] int prior_count = this->add_awaiter();
      DEBUG_CHECK_EQUAL(prior_count, 0, "expected handle to be previously unowned!");
      return BasicTask(this->self(), BasicTask::construction_key{});
    }
 
    TaskFactory take_shared_ownership(int count);
 
    TaskHandle release_ownership() { return this->release_awaiter() ? this->self() : TaskHandle{}; }
 
    constexpr TaskPromiseBase() noexcept = default;
 
    template <TaskPromise Promise>
    [[nodiscard]] TaskHandle initialize_task(std::coroutine_handle<Promise> handle) noexcept
    {
      DEBUG_PRECONDITION(!self_);
      self_ = TaskHandle::from(handle);
      this->add_awaiter();
      TaskRegistry::register_task(self_.coroutine());
#if UNI20_DEBUG_DAG
      TaskRegistry::record_task_dependencies(self_.coroutine(), ReadDependencies, WriteDependencies);
#endif
      return self_;
    }
 
    auto initial_suspend() noexcept
    {
      struct InitialAwaiter
      {
          TaskPromiseBase* promise;
 
          [[nodiscard]] constexpr bool await_ready() noexcept { return false; }
          void await_suspend(std::coroutine_handle<>) noexcept { TaskPromiseBase::note_suspended(promise->self()); }
          constexpr void await_resume() noexcept {}
      };
      return InitialAwaiter{this};
    }
 
 
    auto final_suspend() noexcept
    {
      struct FinalAwaiter
      {
          TaskPromiseBase* promise;
 
          [[nodiscard]] constexpr bool await_ready() noexcept { return false; }
 
          std::coroutine_handle<> await_suspend(std::coroutine_handle<>) noexcept
          {
            auto completed = promise->self();
            auto continuation = std::exchange(promise->continuation_, {});
            auto* continuation_exception = std::exchange(promise->continuation_exception_, nullptr);
            bool cancelled = promise->is_cancel_on_resume();
            auto eptr = promise->get_exception();
            bool const direct_transfer =
                continuation && TaskPromiseBase::can_transfer_directly(completed, continuation);
            TRACE_MODULE(ASYNC, "Final suspend of coroutine", completed.coroutine(), continuation.coroutine(),
                         cancelled);
 
            TaskPromiseBase::destroy_and_track(completed);
 
            TRACE_MODULE(ASYNC, "Destroy is done");
 
            if (cancelled)
            {
              while (continuation)
                continuation = continuation.promise().destroy_with_continuation();
            }
 
            if (continuation)
            {
              if (eptr)
              {
                if (continuation_exception)
                  *continuation_exception = eptr;
                else
                  continuation.promise().set_exception(eptr);
              }
 
              if (!direct_transfer)
              {
                auto continuation_task = continuation.promise().take_ownership();
                BasicTask::reschedule(std::move(continuation_task));
                return std::noop_coroutine();
              }
 
              TaskPromiseBase::note_running(continuation);
              return continuation.coroutine();
            }
            else
              return std::noop_coroutine();
          }
 
          void await_resume() noexcept {}
      };
      return FinalAwaiter{this};
    }
 
    constexpr void return_void() noexcept {}
 
    void unhandled_exception()
    {
      try
      {
        throw;
      }
      catch (task_cancelled const&)
      {
        this->set_cancel_on_resume();
      }
      catch (...)
      {
        bool const originating_failure = !exception_.load(std::memory_order_acquire);
        auto eptr = std::current_exception();
        TaskRegistry::record_unhandled_exception(this->self().coroutine(), eptr, originating_failure);
        this->set_exception(eptr);
        for (auto* node = exception_sinks_head_; node; node = node->next)
          propagate_unhandled_writer_exception(node->epoch.get(), eptr);
      }
    }
 
    // /// \brief Called when the task is ready to run again.
    // /// \param h The coroutine handle to schedule.
    // void notify_ready(TaskHandle handle) { sched_->schedule(handle); }
 
  protected:
    static void validate_scheduler_route(IScheduler* scheduler, TaskRoute route)
    {
      CHECK(scheduler, "task scheduler route cannot be null");
      CHECK(scheduler->accepts_route(route), "scheduler does not accept task route", static_cast<int>(route.domain),
            route.cuda_device.value_or(-1));
    }
 
  private:
    IScheduler* sched_ = nullptr;
    TaskHandle self_{};
 
    friend class BasicTask;
};
 
class AsyncTaskPromise final : public TaskPromiseBase {
  public:
    static constexpr TaskDomain task_domain = TaskDomain::host;
 
    using TaskPromiseBase::TaskPromiseBase;
 
    [[nodiscard]] AsyncTask get_return_object() noexcept
    {
      auto handle = std::coroutine_handle<AsyncTaskPromise>::from_promise(*this);
      return AsyncTask(this->initialize_task(handle));
    }
};
 
class CudaTaskPromise final : public TaskPromiseBase {
  public:
    static constexpr TaskDomain task_domain = TaskDomain::cuda;
 
    using TaskPromiseBase::await_transform;
    using TaskPromiseBase::TaskPromiseBase;
 
    [[nodiscard]] CudaTask get_return_object() noexcept;
 
    [[nodiscard]] std::optional<int> device() const noexcept { return device_; }
 
    template <CudaTaskAwaitable Awaiter> Awaiter&& await_transform(Awaiter&& awaiter) noexcept
    {
      return std::forward<Awaiter>(awaiter);
    }
 
    void bind_device(int device)
    {
      CHECK(device >= 0, "CUDA device ordinal must be non-negative", device);
      CHECK(!this->has_started(), "cannot bind a CUDA task device after the task has started");
      if (device_)
      {
        CHECK_EQUAL(*device_, device, "cannot rebind a CUDA task to another device");
        return;
      }
      if (auto* scheduler = this->scheduler())
      {
        this->validate_scheduler_route(scheduler, TaskRoute{.domain = TaskDomain::cuda, .cuda_device = device});
      }
      device_ = device;
    }
 
    void select_device(int device)
    {
      CHECK(device >= 0, "CUDA device ordinal must be non-negative", device);
      this->validate_scheduler_route(this->scheduler(), TaskRoute{.domain = TaskDomain::cuda, .cuda_device = device});
      device_ = device;
    }
 
  private:
    std::optional<int> device_{};
};
 
inline CudaTaskPromise& cuda_promise(TaskHandle handle) noexcept
{
  CHECK(handle);
  CHECK(handle.domain() == TaskDomain::cuda);
  return static_cast<CudaTaskPromise&>(handle.promise());
}
 
inline TaskRoute task_route(TaskHandle handle) noexcept
{
  CHECK(handle);
  TaskRoute route{.domain = handle.domain()};
  if (handle.domain() == TaskDomain::cuda) route.cuda_device = cuda_promise(handle).device();
  return route;
}
 
inline void TaskPromiseBase::prepare_nested_route(TaskHandle parent, TaskHandle child)
{
  CHECK(parent);
  CHECK(child);
  CHECK(parent.promise().scheduler(), "nested task parent has no scheduler route");
 
  if (!child.promise().scheduler())
  {
    auto* scheduler = parent.promise().scheduler();
    TaskPromiseBase::validate_scheduler_route(scheduler, task_route(child));
    child.promise().sched_ = scheduler;
  }
 
  if (child.domain() == TaskDomain::cuda)
  {
    auto& child_promise = cuda_promise(child);
    if (!child_promise.device() && parent.domain() == TaskDomain::cuda)
    {
      if (auto const parent_device = cuda_promise(parent).device()) child_promise.bind_device(*parent_device);
    }
  }
}
 
inline bool TaskPromiseBase::can_transfer_directly(TaskHandle from, TaskHandle to) noexcept
{
  CHECK(from);
  CHECK(to);
  auto* scheduler = from.promise().scheduler();
  CHECK(scheduler, "running task has no scheduler route");
 
  if (scheduler != to.promise().scheduler()) return false;
  if (from.domain() != to.domain()) return false;
  return scheduler->can_direct_transfer(from, to);
}
 
class TaskFactory {
  public:
    BasicTask take_next()
    {
      DEBUG_PRECONDITION(count_ > 0);
      --count_;
      return BasicTask(handle_, BasicTask::construction_key{});
    }
 
    TaskFactory(TaskFactory&& other) noexcept
        : handle_(std::exchange(other.handle_, {})), count_(std::exchange(other.count_, 0))
    {}
 
    TaskFactory& operator=(TaskFactory&& other) noexcept
    {
      if (this != &other)
      {
        this->release_outstanding();
        handle_ = std::exchange(other.handle_, {});
        count_ = std::exchange(other.count_, 0);
      }
      return *this;
    }
 
    TaskFactory(TaskFactory const&) = delete;
    TaskFactory& operator=(TaskFactory const&) = delete;
 
    ~TaskFactory() noexcept
    {
      DEBUG_TRACE_MODULE(ASYNC, this, handle_.coroutine(), count_);
      this->release_outstanding();
    }
 
  private:
    friend class TaskPromiseBase;
 
    TaskFactory(TaskHandle handle, int count) : handle_(handle), count_(count)
    {
      [[maybe_unused]] int prior_count = handle_.promise().add_awaiter(count);
      DEBUG_CHECK_EQUAL(prior_count, 0, "expected handle to be previously unowned!");
      // if we requested zero references, then we can destroy the handle immediately
      if (count_ == 0)
      {
        TaskPromiseBase::destroy_and_track(handle_);
        handle_ = {};
      }
    }
 
    void release_outstanding() noexcept
    {
      // Return the outstanding references. If this reaches zero, destroy the coroutine.
      if (count_ > 0 && handle_.promise().release_awaiter(count_))
      {
        TaskPromiseBase::destroy_and_track(handle_);
      }
      count_ = 0;
      handle_ = {};
    }
 
    TaskHandle handle_;
    size_t count_;
};
 
inline TaskFactory TaskPromiseBase::take_shared_ownership(int count) { return TaskFactory(this->self(), count); }
 
template <TaskAwaitable A> auto TaskPromiseBase::suspend_task_awaitable(TaskHandle current, A& awaitable)
{
#if UNI20_DEBUG_DAG
  TaskPromiseBase::note_await_dependency(current, awaitable);
#endif
  using await_return_type = decltype(awaitable.await_suspend(std::declval<BasicTask>()));
  if constexpr (std::is_void_v<await_return_type>)
  {
    auto task = current.promise().take_ownership();
    awaitable.await_suspend(std::move(task));
    TaskPromiseBase::note_suspended(current);
    return;
  }
  else if constexpr (std::is_same_v<await_return_type, BasicTask>)
  {
    auto owning_task = current.promise().take_ownership();
    auto task = awaitable.await_suspend(std::move(owning_task));
    return TaskPromiseBase::resolve_await_suspend_result(current, std::move(task));
  }
  else
  {
    static_assert(std::is_same_v<await_return_type, void>,
                  "Unsupported await_suspend() return type: must be void or BasicTask");
  }
}
 
template <TaskFactoryAwaitable A> void TaskPromiseBase::suspend_factory_awaitable(TaskHandle current, A& awaitable)
{
#if UNI20_DEBUG_DAG
  TaskPromiseBase::note_await_dependency(current, awaitable);
#endif
  auto factory = current.promise().take_shared_ownership(awaitable.num_awaiters());
  awaitable.await_suspend(std::move(factory));
  TaskPromiseBase::note_suspended(current);
}
 
template <TaskAwaitable A> struct TaskAwaiter //: public AsyncAwaiter
{
    A awaitable;
    TaskPromiseBase& promise;
 
    [[nodiscard]] bool await_ready() { return awaitable.await_ready(); }
 
    template <TaskPromise Promise> auto await_suspend(std::coroutine_handle<Promise> handle)
    {
      return TaskPromiseBase::suspend_task_awaitable(TaskHandle::from(handle), awaitable);
    }
 
    [[nodiscard]] decltype(auto) await_resume()
    {
#if UNI20_DEBUG_DAG
      TaskPromiseBase::note_await_dependency(promise.self(), awaitable);
#endif
      if constexpr (requires { awaitable.register_exception_sinks(promise); })
      {
        awaitable.register_exception_sinks(promise);
      }
      if constexpr (std::is_lvalue_reference_v<A>)
      {
        return awaitable.await_resume();
      }
      else
      {
        return std::move(awaitable).await_resume();
      }
    }
 
    // decltype(auto) await_resume() &
    // {
    //   // we can call await_resume on a moved awaitable here, because this is the last time
    //   // that we refer to awaitable
    //   return awaitable.await_resume();
    // }
    //
    // decltype(auto) await_resume() &&
    // {
    //   // we can call await_resume on a moved awaitable here, because this is the last time
    //   // that we refer to awaitable
    //   return std::move(awaitable).await_resume();
    // }
 
    // void set_cancel() override final { awaitable.set_cancel(); }
    //
    // void set_exception(std::exception_ptr e) override final { awaitable.set_exception(e); }
};
 
template <typename T> void ProcessCoroutineArgument(TaskPromiseBase* promise, T const& value)
{
  static_cast<void>(promise);
  static_cast<void>(value);
}
 
template <TaskFactoryAwaitable A> struct TaskFactoryAwaiter //: public AsyncAwaiter
{
    A awaitable;
    TaskPromiseBase& promise;
 
    [[nodiscard]] bool await_ready() { return awaitable.await_ready(); }
 
    template <TaskPromise Promise> void await_suspend(std::coroutine_handle<Promise> handle)
    {
      TaskPromiseBase::suspend_factory_awaitable(TaskHandle::from(handle), awaitable);
    }
 
    [[nodiscard]] decltype(auto) await_resume()
    {
#if UNI20_DEBUG_DAG
      TaskPromiseBase::note_await_dependency(promise.self(), awaitable);
#endif
      if constexpr (std::is_lvalue_reference_v<A>)
      {
        return awaitable.await_resume();
      }
      else
      {
        return std::move(awaitable).await_resume();
      }
    }
 
    // void set_cancel() override final { awaitable.set_cancel(); }
    //
    // void set_exception(std::exception_ptr e) override final { awaitable.set_exception(e); }
};
 
template <TaskPromise Promise> TaskHandle TaskHandle::from(std::coroutine_handle<Promise> handle) noexcept
{
  if (!handle) return {};
  return TaskHandle(std::coroutine_handle<>::from_address(handle.address()), std::addressof(handle.promise()),
                    Promise::task_domain);
}
 
inline TaskPromiseBase& TaskHandle::promise() const noexcept
{
  DEBUG_PRECONDITION(promise_ != nullptr);
  return *promise_;
}
 
} // namespace uni20::async
 
#include "async_task_impl.hpp"