<CORE>/ShaderLibrary/ThreadingEmuImpl.hlsl full source
#ifndef THREADING_EMU_IMPL
#define THREADING_EMU_IMPL
// If the user didn't specify a wave size, we assume that their code is "wave size independent" and that they don't
// care which size is actually used. In this case, we automatically select an arbitrary size for them since the
// emulation logic depends on having *some* known size.
#ifndef THREADING_WAVE_SIZE
#define THREADING_WAVE_SIZE 32
#endif
namespace Threading
{
// Currently we only cover scalar types as at the time of writing this utility library we only needed emulation for those.
// Support for vector types is currently not there but can be added as needed (and this comment removed).
groupshared uint g_Scratch[THREADING_BLOCK_SIZE];
#define EMULATED_WAVE_REDUCE(TYPE, OP) \
GroupMemoryBarrierWithGroupSync(); \
g_Scratch[indexG] = asuint(v); \
GroupMemoryBarrierWithGroupSync(); \
[unroll] \
for (uint s = THREADING_WAVE_SIZE / 2u; s > 0u; s >>= 1u) \
{ \
if (indexL < s) \
g_Scratch[indexG] = asuint(as##TYPE(g_Scratch[indexG]) OP as##TYPE(g_Scratch[indexG + s])); \
GroupMemoryBarrierWithGroupSync(); \
} \
return as##TYPE(g_Scratch[offset]); \
#define EMULATED_WAVE_REDUCE_CMP(TYPE, OP) \
GroupMemoryBarrierWithGroupSync(); \
g_Scratch[indexG] = asuint(v); \
GroupMemoryBarrierWithGroupSync(); \
[unroll] \
for (uint s = THREADING_WAVE_SIZE / 2u; s > 0u; s >>= 1u) \
{ \
if (indexL < s) \
g_Scratch[indexG] = asuint(OP(as##TYPE(g_Scratch[indexG]), as##TYPE(g_Scratch[indexG + s]))); \
GroupMemoryBarrierWithGroupSync(); \
} \
return as##TYPE(g_Scratch[offset]); \
#define EMULATED_WAVE_PREFIX(TYPE, OP, FILL_VALUE) \
GroupMemoryBarrierWithGroupSync(); \
g_Scratch[indexG] = asuint(v); \
GroupMemoryBarrierWithGroupSync(); \
[unroll] \
for (uint s = 1u; s < THREADING_WAVE_SIZE; s <<= 1u) \
{ \
TYPE nv = FILL_VALUE; \
if (indexL >= s) \
{ \
nv = as##TYPE(g_Scratch[indexG - s]); \
} \
nv = as##TYPE(g_Scratch[indexG]) OP nv; \
GroupMemoryBarrierWithGroupSync(); \
g_Scratch[indexG] = asuint(nv); \
GroupMemoryBarrierWithGroupSync(); \
} \
TYPE result = FILL_VALUE; \
if (indexL > 0u) \
result = as##TYPE(g_Scratch[indexG - 1]); \
return result; \
uint Wave::GetIndex() { return indexW; }
void Wave::Init(uint groupIndex)
{
indexG = groupIndex;
indexW = indexG / THREADING_WAVE_SIZE;
indexL = indexG & (THREADING_WAVE_SIZE - 1);
offset = indexW * THREADING_WAVE_SIZE;
}
// WARNING:
// These emulated functions do not emulate the execution mask.
// So they WILL produce incorrect results if you have divergent lanes.
#define DEFINE_API_FOR_TYPE(TYPE) \
bool Wave::AllEqual(TYPE v) { return AllTrue(ReadLaneFirst(v) == v); } \
TYPE Wave::Product(TYPE v) { EMULATED_WAVE_REDUCE(TYPE, *) } \
TYPE Wave::Sum(TYPE v) { EMULATED_WAVE_REDUCE(TYPE, +) } \
TYPE Wave::Max(TYPE v) { EMULATED_WAVE_REDUCE_CMP(TYPE, max) } \
TYPE Wave::Min(TYPE v) { EMULATED_WAVE_REDUCE_CMP(TYPE, min) } \
TYPE Wave::InclusivePrefixSum (TYPE v) { return PrefixSum(v) + v; } \
TYPE Wave::InclusivePrefixProduct (TYPE v) { return PrefixProduct(v) * v; } \
TYPE Wave::PrefixSum (TYPE v) { EMULATED_WAVE_PREFIX(TYPE, +, (TYPE)0) } \
TYPE Wave::PrefixProduct (TYPE v) { EMULATED_WAVE_PREFIX(TYPE, *, (TYPE)1) } \
TYPE Wave::ReadLaneAt(TYPE v, uint i) { GroupMemoryBarrierWithGroupSync(); g_Scratch[indexG] = asuint(v); GroupMemoryBarrierWithGroupSync(); return as##TYPE(g_Scratch[offset + i]); } \
TYPE Wave::ReadLaneFirst(TYPE v) { return ReadLaneAt(v, 0u); } \
// Currently just support scalars.
DEFINE_API_FOR_TYPE(uint)
DEFINE_API_FOR_TYPE(int)
DEFINE_API_FOR_TYPE(float)
// The following emulated functions need only be declared once.
uint Wave::GetLaneCount() { return THREADING_WAVE_SIZE; }
uint Wave::GetLaneIndex() { return indexL; }
bool Wave::IsFirstLane() { return indexL == 0u; }
bool Wave::AllTrue(bool v) { return And(v) != 0u; }
bool Wave::AnyTrue(bool v) { return Or (v) != 0u; }
uint Wave::PrefixCountBits(bool v) { return PrefixSum((uint)v); }
uint Wave::And(uint v) { EMULATED_WAVE_REDUCE(uint, &) }
uint Wave::Or (uint v) { EMULATED_WAVE_REDUCE(uint, |) }
uint Wave::Xor(uint v) { EMULATED_WAVE_REDUCE(uint, ^) }
uint4 Wave::Ballot(bool v)
{
uint indexDw = indexL % 32u;
uint offsetDw = (indexL / 32u) * 32u;
uint indexScratch = offset + offsetDw + indexDw;
GroupMemoryBarrierWithGroupSync();
g_Scratch[indexG] = v << indexDw;
GroupMemoryBarrierWithGroupSync();
[unroll]
for (uint s = min(THREADING_WAVE_SIZE / 2u, 16u); s > 0u; s >>= 1u)
{
if (indexDw < s)
g_Scratch[indexScratch] = g_Scratch[indexScratch] | g_Scratch[indexScratch + s];
GroupMemoryBarrierWithGroupSync();
}
uint4 result = uint4(g_Scratch[offset], 0, 0, 0);
#if THREADING_WAVE_SIZE > 32
result.y = g_Scratch[offset + 32];
#endif
#if THREADING_WAVE_SIZE > 64
result.z = g_Scratch[offset + 64];
#endif
#if THREADING_WAVE_SIZE > 96
result.w = g_Scratch[offset + 96];
#endif
return result;
}
uint Wave::CountBits(bool v)
{
uint4 ballot = Ballot(v);
uint result = countbits(ballot.x);
#if THREADING_WAVE_SIZE > 32
result += countbits(ballot.y);
#endif
#if THREADING_WAVE_SIZE > 64
result += countbits(ballot.z);
#endif
#if THREADING_WAVE_SIZE > 96
result += countbits(ballot.w);
#endif
return result;
}
#define EMULATED_GROUP_REDUCE(TYPE, OP) \
GroupMemoryBarrierWithGroupSync(); \
g_Scratch[groupIndex] = asuint(v); \
GroupMemoryBarrierWithGroupSync(); \
[unroll] \
for (uint s = THREADING_BLOCK_SIZE / 2u; s > 0u; s >>= 1u) \
{ \
if (groupIndex < s) \
g_Scratch[groupIndex] = asuint(as##TYPE(g_Scratch[groupIndex]) OP as##TYPE(g_Scratch[groupIndex + s])); \
GroupMemoryBarrierWithGroupSync(); \
} \
return as##TYPE(g_Scratch[0]); \
#define EMULATED_GROUP_REDUCE_CMP(TYPE, OP) \
GroupMemoryBarrierWithGroupSync(); \
g_Scratch[groupIndex] = asuint(v); \
GroupMemoryBarrierWithGroupSync(); \
[unroll] \
for (uint s = THREADING_BLOCK_SIZE / 2u; s > 0u; s >>= 1u) \
{ \
if (groupIndex < s) \
g_Scratch[groupIndex] = asuint(OP(as##TYPE(g_Scratch[groupIndex]), as##TYPE(g_Scratch[groupIndex + s]))); \
GroupMemoryBarrierWithGroupSync(); \
} \
return as##TYPE(g_Scratch[0]); \
#define EMULATED_GROUP_PREFIX(TYPE, OP, FILL_VALUE) \
GroupMemoryBarrierWithGroupSync(); \
g_Scratch[groupIndex] = asuint(v); \
GroupMemoryBarrierWithGroupSync(); \
[unroll] \
for (uint s = 1u; s < THREADING_BLOCK_SIZE; s <<= 1u) \
{ \
TYPE nv = FILL_VALUE; \
if (groupIndex >= s) \
{ \
nv = as##TYPE(g_Scratch[groupIndex - s]); \
} \
nv = as##TYPE(g_Scratch[groupIndex]) OP nv; \
GroupMemoryBarrierWithGroupSync(); \
g_Scratch[groupIndex] = asuint(nv); \
GroupMemoryBarrierWithGroupSync(); \
} \
TYPE result = FILL_VALUE; \
if (groupIndex > 0u) \
result = as##TYPE(g_Scratch[groupIndex - 1]); \
return result; \
uint Group::GetWaveCount()
{
return THREADING_BLOCK_SIZE / THREADING_WAVE_SIZE;
}
#define DEFINE_API_FOR_TYPE_GROUP(TYPE) \
bool Group::AllEqual(TYPE v) { return AllTrue(ReadThreadFirst(v) == v); } \
TYPE Group::Product(TYPE v) { EMULATED_GROUP_REDUCE(TYPE, *) } \
TYPE Group::Sum(TYPE v) { EMULATED_GROUP_REDUCE(TYPE, +) } \
TYPE Group::Max(TYPE v) { EMULATED_GROUP_REDUCE_CMP(TYPE, max) } \
TYPE Group::Min(TYPE v) { EMULATED_GROUP_REDUCE_CMP(TYPE, min) } \
TYPE Group::InclusivePrefixSum (TYPE v) { return PrefixSum(v) + v; } \
TYPE Group::InclusivePrefixProduct (TYPE v) { return PrefixProduct(v) * v; } \
TYPE Group::PrefixSum (TYPE v) { EMULATED_GROUP_PREFIX(TYPE, +, (TYPE)0) } \
TYPE Group::PrefixProduct (TYPE v) { EMULATED_GROUP_PREFIX(TYPE, *, (TYPE)1) } \
TYPE Group::ReadThreadAt(TYPE v, uint i) { GroupMemoryBarrierWithGroupSync(); g_Scratch[groupIndex] = asuint(v); GroupMemoryBarrierWithGroupSync(); return as##TYPE(g_Scratch[i]); } \
TYPE Group::ReadThreadFirst(TYPE v) { return ReadThreadAt(v, 0u); } \
TYPE Group::ReadThreadShuffle(TYPE v, uint i) { return ReadThreadAt(v, i); } \
// Currently just support scalars.
DEFINE_API_FOR_TYPE_GROUP(uint)
DEFINE_API_FOR_TYPE_GROUP(int)
DEFINE_API_FOR_TYPE_GROUP(float)
// The following emulated functions need only be declared once.
uint Group::GetThreadCount() { return THREADING_BLOCK_SIZE; }
uint Group::GetThreadIndex() { return groupIndex; }
bool Group::IsFirstThread() { return groupIndex == 0u; }
bool Group::AllTrue(bool v) { return And(v) != 0u; }
bool Group::AnyTrue(bool v) { return Or (v) != 0u; }
uint Group::PrefixCountBits(bool v) { return PrefixSum((uint)v); }
uint Group::And(uint v) { EMULATED_GROUP_REDUCE(uint, &) }
uint Group::Or (uint v) { EMULATED_GROUP_REDUCE(uint, |) }
uint Group::Xor(uint v) { EMULATED_GROUP_REDUCE(uint, ^) }
GroupBallot Group::Ballot(bool v)
{
uint indexDw = groupIndex % 32u;
uint offsetDw = (groupIndex / 32u) * 32u;
uint indexScratch = offsetDw + indexDw;
GroupMemoryBarrierWithGroupSync();
g_Scratch[groupIndex] = v << indexDw;
GroupMemoryBarrierWithGroupSync();
[unroll]
for (uint s = min(THREADING_BLOCK_SIZE / 2u, 16u); s > 0u; s >>= 1u)
{
if (indexDw < s)
g_Scratch[indexScratch] = g_Scratch[indexScratch] | g_Scratch[indexScratch + s];
GroupMemoryBarrierWithGroupSync();
}
GroupBallot ballot = (GroupBallot)0;
// Explicitly mark this loop as "unroll" to avoid warnings about assigning to an array reference
[unroll]
for (uint dwordIndex = 0; dwordIndex < _THREADING_GROUP_BALLOT_DWORDS; ++dwordIndex)
{
ballot.dwords[dwordIndex] = g_Scratch[dwordIndex * 32];
}
return ballot;
}
uint Group::CountBits(bool v)
{
return Ballot(v).CountBits();
}
}
#endif