19 namespace devi = devirtualize_parameters;
25 namespace CustomMF_presets {
66 template<
typename Fn,
typename... ParamTypes>
72 DeviMode::Span | DeviMode::Single | DeviMode::Range,
73 sizeof...(ParamTypes)>());
86 template<
typename Fn,
typename... ParamTypes>
92 DeviMode::Span | DeviMode::Single | DeviMode::Range,
94 sizeof...(ParamTypes),
109 template<
typename MaskT,
typename... Args,
typename... ParamTags,
size_t...
I,
typename ElementFn>
111 std::index_sequence<I...> ,
112 ElementFn element_fn,
116 Args &&__restrict... args)
119 element_fn([&]() -> decltype(
auto) {
120 using ParamTag =
typename TypeSequence<ParamTags...>::template at_index<I>;
136 namespace materialize_detail {
153 template<
typename... ParamTags,
typename ElementFn,
typename... Chunks>
155 const ElementFn element_fn,
158 Chunks &&__restrict... chunks)
162 const int64_t in_i = in_mask[i];
163 const int64_t out_i = out_mask[i];
164 element_fn([&]() -> decltype(
auto) {
165 using ParamTag = ParamTags;
171 return &chunks[out_i];
182 template<
typename... ParamTags,
size_t...
I,
typename ElementFn,
typename... Args>
184 std::index_sequence<I...> ,
185 const ElementFn element_fn,
194 static constexpr
int64_t MaxChunkSize = 32;
199 std::tuple<TypedBuffer<typename ParamTags::base_type, MaxChunkSize>...> buffers_owner;
202 std::tuple<MutableSpan<typename ParamTags::base_type>...> buffers;
205 std::tuple<ArgInfo<ParamTags>...> args_info;
210 typedef ParamTags ParamTag;
211 typedef typename ParamTag::base_type
T;
218 MutableSpan<T> in_chunk{std::get<I>(buffers_owner).ptr(), buffer_size};
221 std::get<I>(buffers) = in_chunk;
233 for (
int64_t chunk_start = 0; chunk_start < mask_size; chunk_start += MaxChunkSize) {
234 const IndexMask sliced_mask =
mask.slice(chunk_start, MaxChunkSize);
236 const bool sliced_mask_is_range = sliced_mask.
is_range();
247 using ParamTag = ParamTags;
248 using T =
typename ParamTag::base_type;
253 return Span<T>(std::get<I>(buffers));
257 if (sliced_mask_is_range) {
285 typedef ParamTags ParamTag;
286 typedef typename ParamTag::base_type
T;
290 T *in_chunk = std::get<I>(buffers_owner).ptr();
301 typedef ParamTags ParamTag;
302 typedef typename ParamTag::base_type
T;
323 template<
typename ElementFn,
typename ExecPreset = CustomMF_presets::Materialized>
325 ElementFn element_fn,
339 template<
typename ElementFn,
typename ExecPreset,
size_t...
I>
341 ExecPreset exec_preset,
344 std::index_sequence<I...> )
346 std::tuple<
typename ParamTags::array_type...> retrieved_params;
350 typedef typename TagsSequence::template at_index<I> ParamTag;
351 typedef typename ParamTag::base_type
T;
354 std::get<I>(retrieved_params) =
params.readonly_single_input<
T>(
I);
357 std::get<I>(retrieved_params) =
params.uninitialized_single_output<
T>(
I);
362 auto array_executor = [&](
auto &&...args) {
366 std::forward<decltype(args)>(args)...);
370 bool executed_devirtualized =
false;
371 if constexpr (ExecPreset::use_devirtualization) {
374 array_executor, &
mask, [&] {
return &std::get<I>(retrieved_params); }()...};
375 exec_preset.try_devirtualize(devirtualizer);
376 executed_devirtualized = devirtualizer.executed();
381 if (!executed_devirtualized) {
385 return &std::get<I>(retrieved_params);
393 std::get<I>(retrieved_params)...);
398 template<
size_t...
I>
400 std::index_sequence<I...> )
405 typedef typename TagsSequence::template at_index<I> ParamTag;
425 template<
typename In1,
typename Out1>
427 MFParamTag<MFParamCategory::SingleOutput, Out1>> {
429 template<
typename ElementFn,
typename ExecPreset = CustomMF_presets::Materialized>
431 ElementFn element_fn,
436 [element_fn](const In1 &in1, Out1 *out1) {
new (out1) Out1(element_fn(in1)); },
448 template<
typename In1,
typename In2,
typename Out1>
450 MFParamTag<MFParamCategory::SingleInput, In2>,
451 MFParamTag<MFParamCategory::SingleOutput, Out1>> {
453 template<
typename ElementFn,
typename ExecPreset = CustomMF_presets::Materialized>
455 ElementFn element_fn,
461 [element_fn](const In1 &in1, const In2 &in2, Out1 *out1) {
462 new (out1) Out1(element_fn(in1, in2));
476 template<
typename In1,
typename In2,
typename In3,
typename Out1>
478 MFParamTag<MFParamCategory::SingleInput, In2>,
479 MFParamTag<MFParamCategory::SingleInput, In3>,
480 MFParamTag<MFParamCategory::SingleOutput, Out1>> {
482 template<
typename ElementFn,
typename ExecPreset = CustomMF_presets::Materialized>
484 ElementFn element_fn,
491 [element_fn](const In1 &in1, const In2 &in2, const In3 &in3, Out1 *out1) {
492 new (out1) Out1(element_fn(in1, in2, in3));
507 template<
typename In1,
typename In2,
typename In3,
typename In4,
typename Out1>
509 MFParamTag<MFParamCategory::SingleInput, In2>,
510 MFParamTag<MFParamCategory::SingleInput, In3>,
511 MFParamTag<MFParamCategory::SingleInput, In4>,
512 MFParamTag<MFParamCategory::SingleOutput, Out1>> {
514 template<
typename ElementFn,
typename ExecPreset = CustomMF_presets::Materialized>
516 ElementFn element_fn,
525 const In1 &in1, const In2 &in2, const In3 &in3, const In4 &in4, Out1 *out1) {
526 new (out1) Out1(element_fn(in1, in2, in3, in4));
552 template<
typename ElementFuncT>
558 template<
typename ElementFuncT>
static FunctionT
create_function(ElementFuncT element_fn)
561 mask.to_best_mask_type([&](
const auto &
mask) {
572 function_(
mask, mut1);
632 mask.to_best_mask_type([&](
const auto &
mask) {
647 if (other1 !=
nullptr) {
648 return value_ == other1->value_;
652 if (other2 !=
nullptr) {
654 if (
type == other2->type_) {
655 return type.is_equal_or_false(
static_cast<const void *
>(&value_), other2->value_);
_GL_VOID GLfloat value _GL_VOID_RET _GL_VOID const GLuint GLboolean *residences _GL_BOOL_RET _GL_VOID GLsizei GLfloat GLfloat GLfloat GLfloat const GLubyte *bitmap _GL_VOID_RET _GL_VOID GLenum type
IndexRange as_range() const
constexpr int64_t size() const
constexpr int64_t size() const
constexpr T * data() const
constexpr Span slice(int64_t start, int64_t size) const
constexpr bool is_empty() const
T get_internal_single() const
void materialize_compressed_to_uninitialized(IndexMask mask, MutableSpan< T > r_span) const
Span< T > get_internal_span() const
void try_execute_devirtualized(DeviModeSequence< AllowedModes... >)
void call(IndexMask mask, MFParams params, MFContext UNUSED(context)) const override
CustomMF_Constant(U &&value)
uint64_t hash() const override
bool equals(const MultiFunction &other) const override
void call(IndexMask mask, MFParams params, MFContext context) const override
CustomMF_DefaultOutput(Span< MFDataType > input_types, Span< MFDataType > output_types)
CustomMF_GenericConstantArray(GSpan array)
void call(IndexMask mask, MFParams params, MFContext context) const override
CustomMF_GenericConstant(const CPPType &type, const void *value, bool make_value_copy)
~CustomMF_GenericConstant()
void call(IndexMask mask, MFParams params, MFContext context) const override
uint64_t hash() const override
bool equals(const MultiFunction &other) const override
void call(IndexMask mask, MFParams params, MFContext context) const override
CustomMF_GenericCopy(MFDataType data_type)
CustomMF_SI_SI_SI_SI_SO(const char *name, ElementFn element_fn, ExecPreset exec_preset=CustomMF_presets::Materialized())
CustomMF_SI_SI_SI_SO(const char *name, ElementFn element_fn, ExecPreset exec_preset=CustomMF_presets::Materialized())
CustomMF_SI_SI_SO(const char *name, ElementFn element_fn, ExecPreset exec_preset=CustomMF_presets::Materialized())
CustomMF_SI_SO(const char *name, ElementFn element_fn, ExecPreset exec_preset=CustomMF_presets::Materialized())
void call(IndexMask mask, MFParams params, MFContext UNUSED(context)) const override
CustomMF_SM(const char *name, ElementFuncT element_fn)
static FunctionT create_function(ElementFuncT element_fn)
CustomMF_SM(const char *name, FunctionT function)
void call(IndexMask mask, MFParams params, MFContext UNUSED(context)) const override
CustomMF(const char *name, ElementFn element_fn, ExecPreset exec_preset=CustomMF_presets::Materialized())
static void execute(ElementFn element_fn, ExecPreset exec_preset, IndexMask mask, MFParams params, std::index_sequence< I... >)
static void add_signature_parameters(MFSignatureBuilder &signature, std::index_sequence< I... >)
StringRefNull name() const
void set_signature(const MFSignature *signature)
const MFSignature & signature() const
SyclQueue void void size_t num_bytes void
ccl_global KernelShaderEvalInput ccl_global float * output
ccl_device_inline float4 mask(const int4 &mask, const float4 &a)
void execute_array(TypeSequence< ParamTags... >, std::index_sequence< I... >, ElementFn element_fn, MaskT mask, Args &&__restrict... args)
void execute_materialized_impl(TypeSequence< ParamTags... >, const ElementFn element_fn, const IndexRange in_mask, const IndexMask out_mask, Chunks &&__restrict... chunks)
void execute_materialized(TypeSequence< ParamTags... >, std::index_sequence< I... >, const ElementFn element_fn, const IndexMask mask, Args &&...args)
static const int chunk_size
decltype(detail::make_value_sequence_impl< T, Value >(std::make_index_sequence< Size >())) make_value_sequence
decltype(detail::make_two_value_sequence_impl< T, Value1, Value2 >(ValueSequence< size_t, Value1Indices... >(), std::make_index_sequence< Size >())) make_two_value_sequence
uint64_t get_default_hash(const T &v)
void uninitialized_fill_n(T *dst, int64_t n, const T &value)
void destruct_n(T *ptr, int64_t n)
unsigned __int64 uint64_t
static constexpr size_t size() noexcept
static constexpr FallbackMode fallback_mode
static constexpr bool use_devirtualization
void try_devirtualize(devi::Devirtualizer< Fn, ParamTypes... > &devirtualizer)
static constexpr bool use_devirtualization
static constexpr FallbackMode fallback_mode
static constexpr FallbackMode fallback_mode
static constexpr bool use_devirtualization
static constexpr FallbackMode fallback_mode
static constexpr bool use_devirtualization
void try_devirtualize(devi::Devirtualizer< Fn, ParamTypes... > &devirtualizer)
Span< typename ParamTag::base_type > internal_span