11#import <Metal/Metal.h>
24 template<
bool SAFE_MATH=false>
30 id<MTLCommandQueue> queue;
32 std::unordered_map<graph::leaf_node<float, SAFE_MATH> *, id<MTLBuffer>> kernel_arguments;
34 std::unordered_map<void *, id<MTLTexture>> texture_arguments;
36 id<MTLCommandBuffer> command_buffer;
38 id<MTLLibrary> library;
40 std::unordered_map<std::string, std::vector<MTLMutability>> bufferMutability;
57 return MTLCopyAllDevices().count;
73 device([MTLCopyAllDevices() objectAtIndex:index]),
74 queue([device newCommandQueue]) {}
83 void compile(
const std::string kernel_source,
84 std::vector<std::string> names,
85 const bool add_reduction=
false) {
87 library = [device newLibraryWithSource:[NSString stringWithCString:kernel_source.c_str()
88 encoding:NSUTF8StringEncoding]
97 std::cout <<
"Metal GPU info." << std::endl
98 <<
" Max thread group memory : " << device.maxThreadgroupMemoryLength << std::endl
99 <<
" Max thread per group : " << device.maxThreadsPerThreadgroup.width << std::endl
100 <<
" Device name : " << [device.name cStringUsingEncoding:NSString.defaultCStringEncoding] << std::endl
101 <<
" Architecture : " << [device.architecture.name cStringUsingEncoding:NSString.defaultCStringEncoding] << std::endl
102 <<
" Max buffer length : " << device.maxBufferLength << std::endl
103 <<
" Max working set : " << device.recommendedMaxWorkingSetSize << std::endl;
125 const size_t num_rays,
130 MTLFunctionDescriptor *funcDesc = [MTLFunctionDescriptor
new];
131 funcDesc.options = MTLFunctionOptionNone;
132 funcDesc.name = [NSString stringWithCString:kernel_name.c_str()
133 encoding:NSUTF8StringEncoding];
135 id<MTLFunction> function = [library newFunctionWithDescriptor:funcDesc
138 MTLComputePipelineDescriptor *compute = [MTLComputePipelineDescriptor
new];
139 compute.threadGroupSizeIsMultipleOfThreadExecutionWidth = YES;
140 compute.computeFunction = function;
141 compute.maxTotalThreadsPerThreadgroup = 1024;
142 for (
size_t i = 0, ie = bufferMutability[kernel_name].size(); i < ie; i++) {
143 compute.buffers[i].mutability = bufferMutability[kernel_name][i];
146 id<MTLComputePipelineState> pipeline = [device newComputePipelineStateWithDescriptor:compute
147 options:MTLPipelineOptionNone
155 std::vector<id<MTLBuffer>> buffers;
156 std::unordered_set<graph::leaf_node<float, SAFE_MATH> *> needed_buffers;
158 const size_t buffer_element_size =
sizeof(float);
160 if (!kernel_arguments.contains(input.get())) {
162 kernel_arguments[input.get()] = [device newBufferWithBytes:buffer.data()
163 length:buffer.size()*buffer_element_size
164 options:MTLResourceStorageModeShared];
166 if (!needed_buffers.contains(input.get())) {
167 buffers.push_back(kernel_arguments[input.get()]);
168 needed_buffers.insert(input.get());
173 if (!kernel_arguments.contains(
output.get())) {
174 kernel_arguments[
output.get()] = [device newBufferWithLength:num_rays*
sizeof(float)
175 options:MTLResourceStorageModeShared];
177 if (!needed_buffers.contains(
output.get())) {
178 buffers.push_back(kernel_arguments[
output.get()]);
179 needed_buffers.insert(
output.get());
184 if (!kernel_arguments.contains(atomic.get())) {
185 kernel_arguments[atomic.get()] = [device newBufferWithLength:atomic->size()*buffer_element_size
186 options:MTLResourceStorageModeShared];
188 if (!needed_buffers.contains(atomic.get())) {
189 buffers.push_back(kernel_arguments[atomic.get()]);
190 needed_buffers.insert(atomic.get());
194 if (!kernel_arguments.contains(state.get())) {
195 kernel_arguments[state.get()] = [device newBufferWithBytes:state->data()
196 length:state->get_size_bytes()
197 options:MTLResourceCPUCacheModeWriteCombined |
198 MTLResourceStorageModeShared |
199 MTLResourceHazardTrackingModeUntracked];
201 buffers.push_back(kernel_arguments[state.get()]);
204 std::vector<id<MTLTexture>> textures;
205 command_buffer = [queue commandBuffer];
206 id<MTLBlitCommandEncoder> encoder = [command_buffer blitCommandEncoder];
207 for (
auto &[data, size] : tex1d_list) {
208 if (!texture_arguments.contains(data)) {
209 MTLTextureDescriptor *descriptor = [MTLTextureDescriptor
new];
210 descriptor.textureType = MTLTextureType1D;
211 descriptor.pixelFormat = MTLPixelFormatR32Float;
212 descriptor.width = size;
213 descriptor.storageMode = MTLStorageModeShared;
214 descriptor.cpuCacheMode = MTLCPUCacheModeWriteCombined;
215 descriptor.hazardTrackingMode = MTLHazardTrackingModeUntracked;
216 descriptor.usage = MTLTextureUsageShaderRead;
217 texture_arguments[data] = [device newTextureWithDescriptor:descriptor];
218 [texture_arguments[data] replaceRegion:MTLRegionMake1D(0, size)
220 withBytes:
reinterpret_cast<float *
> (data)
223 [encoder optimizeContentsForGPUAccess:texture_arguments[data]];
225 textures.push_back(texture_arguments[data]);
227 for (
auto &[data, size] : tex2d_list) {
228 if (!texture_arguments.contains(data)) {
229 MTLTextureDescriptor *descriptor = [MTLTextureDescriptor
new];
230 descriptor.textureType = MTLTextureType2D;
231 descriptor.pixelFormat = MTLPixelFormatR32Float;
232 descriptor.width = size[1];
233 descriptor.height = size[0];
234 descriptor.storageMode = MTLStorageModeShared;
235 descriptor.cpuCacheMode = MTLCPUCacheModeWriteCombined;
236 descriptor.hazardTrackingMode = MTLHazardTrackingModeUntracked;
237 descriptor.usage = MTLTextureUsageShaderRead;
238 texture_arguments[data] = [device newTextureWithDescriptor:descriptor];
239 [texture_arguments[data] replaceRegion:MTLRegionMake2D(0, 0, size[1], size[0])
241 withBytes:
reinterpret_cast<float *
> (data)
242 bytesPerRow:4*size[1]];
244 [encoder optimizeContentsForGPUAccess:texture_arguments[data]];
246 textures.push_back(texture_arguments[data]);
248 [encoder endEncoding];
249 [command_buffer commit];
251 std::vector<NSUInteger> offsets(buffers.size(), 0);
252 NSRange range = NSMakeRange(0, buffers.size());
253 NSRange tex_range = NSMakeRange(0, textures.size());
255 NSUInteger total_parallel = state.get() ? state->size() : num_rays;
256 NSUInteger thread_width = pipeline.threadExecutionWidth;
257 NSUInteger threads_per_group = total_parallel < pipeline.maxTotalThreadsPerThreadgroup ?
259 pipeline.maxTotalThreadsPerThreadgroup;
260 NSUInteger thread_groups = total_parallel/threads_per_group + (total_parallel%threads_per_group ? 1 : 0);
263 std::cout <<
" Kernel name : " << kernel_name << std::endl
264 <<
" Thread execution width : " << thread_width << std::endl
265 <<
" Threads per group : " << threads_per_group << std::endl
266 <<
" Number of groups : " << thread_groups << std::endl
267 <<
" Total problem size : " << threads_per_group*thread_groups << std::endl
268 <<
" Total parallel size : " << total_parallel << std::endl
269 <<
" Current allocation size : " << device.currentAllocatedSize << std::endl;
274 return [
this, num_rays, pipeline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures
275#ifdef PROFILE_KERNELS
279 command_buffer = [queue commandBuffer];
280 for (NSUInteger i = 0, ie = thread_groups*threads_per_group; i < num_rays; i += ie) {
281 id<MTLComputeCommandEncoder> encoder = [command_buffer computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial];
283 [encoder setComputePipelineState:pipeline];
284 [encoder setBuffers:buffers.data()
285 offsets:offsets.data()
288 length:
sizeof(uint32_t)
289 atIndex:buffers.size()];
290 [encoder setTextures:textures.data()
291 withRange:tex_range];
293 [encoder dispatchThreadgroups:MTLSizeMake(thread_groups, 1, 1)
294 threadsPerThreadgroup:MTLSizeMake(threads_per_group, 1, 1)];
295 [encoder endEncoding];
297#ifdef PROFILE_KERNELS
298 [command_buffer addCompletedHandler:[kernel_name](id<MTLCommandBuffer> commandBuffer) {
299 std::cout << std::endl <<
" " << kernel_name <<
" : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime <<
" s" << std::endl << std::endl;
302 [command_buffer commit];
305 return [
this, pipeline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures
306#ifdef PROFILE_KERNELS
310 command_buffer = [queue commandBuffer];
311 id<MTLComputeCommandEncoder> encoder = [command_buffer computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial];
313 [encoder setComputePipelineState:pipeline];
314 [encoder setBuffers:buffers.data()
315 offsets:offsets.data()
317 [encoder setTextures:textures.data()
318 withRange:tex_range];
320 [encoder dispatchThreadgroups:MTLSizeMake(thread_groups, 1, 1)
321 threadsPerThreadgroup:MTLSizeMake(threads_per_group, 1, 1)];
322 [encoder endEncoding];
323#ifdef PROFILE_KERNELS
324 [command_buffer addCompletedHandler:[kernel_name](id<MTLCommandBuffer> commandBuffer) {
325 std::cout << std::endl <<
" " << kernel_name <<
" : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime <<
" s" << std::endl << std::endl;
328 [command_buffer commit];
341 std::function<
void(
void)> run) {
344 MTLFunctionDescriptor *funcDesc = [MTLFunctionDescriptor
new];
345 funcDesc.options = MTLFunctionOptionNone;
346 funcDesc.name =
@"max_reduction";
348 id<MTLFunction> function = [library newFunctionWithDescriptor:funcDesc
351 MTLComputePipelineDescriptor *compute = [MTLComputePipelineDescriptor
new];
352 compute.threadGroupSizeIsMultipleOfThreadExecutionWidth = YES;
353 compute.computeFunction = function;
354 compute.maxTotalThreadsPerThreadgroup = 1024;
355 compute.buffers[0].mutability = MTLMutabilityImmutable;
357 id<MTLComputePipelineState> max_state = [device newComputePipelineStateWithDescriptor:compute
358 options:MTLPipelineOptionNone
365 id<MTLBuffer> result = [device newBufferWithLength:
sizeof(float)
366 options:MTLResourceStorageModeShared];
368 id<MTLBuffer> buffer = kernel_arguments[argument.get()];
370 NSUInteger threads_per_group = max_state.maxTotalThreadsPerThreadgroup;
371 NSUInteger thread_width = max_state.threadExecutionWidth;
373 std::cout <<
" Kernel name : max_reduction" << std::endl;
374 std::cout <<
" Thread execution width : " << thread_width << std::endl;
375 std::cout <<
" Threads per group : " << threads_per_group << std::endl;
376 std::cout <<
" Number of groups : " << 1 << std::endl;
377 std::cout <<
" Total problem size : " << threads_per_group*1 << std::endl;
380 return [
this, run, buffer, result, max_state] ()
mutable {
382 command_buffer = [queue commandBuffer];
384 id<MTLComputeCommandEncoder> encoder = [command_buffer computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial];
386 [encoder setComputePipelineState:max_state];
387 [encoder setBuffer:buffer offset:0 atIndex:0];
388 [encoder setBuffer:result offset:0 atIndex:1];
389 [encoder dispatchThreadgroups:MTLSizeMake(1, 1, 1)
390 threadsPerThreadgroup:MTLSizeMake(1024, 1, 1)];
391 [encoder endEncoding];
393 [command_buffer commit];
394 [command_buffer waitUntilCompleted];
396 return static_cast<float *
> (result.contents)[0];
407 std::vector<id<MTLBuffer>> buffers;
408 for (
auto &input : inputs) {
409 if (!kernel_arguments.contains(input.get())) {
410 kernel_arguments[input.get()] = [device newBufferWithBytes:input->data()
411 length:input->size()*
sizeof(float)
412 options:MTLResourceStorageModeShared];
414 buffers.push_back(kernel_arguments[input.get()]);
417 std::vector<NSRange> ranges;
418 for (id<MTLBuffer> buffer : buffers) {
419 ranges.push_back(NSMakeRange(0, buffer.length));
422 return [
this, buffers, ranges] ()
mutable {
423 command_buffer = [queue commandBuffer];
424 id<MTLBlitCommandEncoder> encoder = [command_buffer blitCommandEncoder];
426 for (
size_t i = 0, ie = buffers.size(); i < ie; i++) {
427 [encoder fillBuffer:buffers[i]
431 [encoder endEncoding];
432#ifdef PROFILE_KERNELS
433 [command_buffer addCompletedHandler:[](id<MTLCommandBuffer> commandBuffer) {
434 std::cout << std::endl <<
" zero buffer : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime <<
" s" << std::endl << std::endl;
437 [command_buffer commit];
448 std::vector<id<MTLBuffer>> sources;
449 std::vector<id<MTLBuffer>> destinations;
451 for (
auto &[out, in] : setters) {
452 if (!kernel_arguments.contains(in.get())) {
453 kernel_arguments[in.get()] = [device newBufferWithBytes:in->data()
454 length:in->size()*
sizeof(float)
455 options:MTLResourceStorageModeShared];
457 destinations.push_back(kernel_arguments[in.get()]);
459 if (!kernel_arguments.contains(out.get())) {
460 kernel_arguments[out.get()] = [device newBufferWithBytes:out->data()
461 length:out->size()*
sizeof(float)
462 options:MTLResourceStorageModeShared];
464 sources.push_back(kernel_arguments[out.get()]);
467 return [
this, sources, destinations] ()
mutable {
468 command_buffer = [queue commandBuffer];
469 id<MTLBlitCommandEncoder> encoder = [command_buffer blitCommandEncoder];
471 for (
size_t i = 0, ie = sources.size(); i < ie; i++) {
472 [encoder copyFromBuffer:sources[i]
474 toBuffer:destinations[i]
476 size:sources[i].length];
478 [encoder endEncoding];
479#ifdef PROFILE_KERNELS
480 [command_buffer addCompletedHandler:[](id<MTLCommandBuffer> commandBuffer) {
481 std::cout << std::endl <<
" copy buffer : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime <<
" s" << std::endl << std::endl;
484 [command_buffer commit];
492 MTLCompileOptions *options = [MTLCompileOptions
new];
493 options.mathMode = MTLMathModeFast;
494 options.mathFloatingPointFunctions = MTLMathFloatingPointFunctionsFast;
495 options.optimizationLevel = MTLLibraryOptimizationLevelDefault;
496 options.languageVersion = MTLLanguageVersion3_2;
504 command_buffer = [queue commandBuffer];
506 [command_buffer commit];
507 [command_buffer waitUntilCompleted];
516 std::function<void(
void)>
run_function(std::function<
void(
void)> callback) {
517 return [
this, callback]() {
518 command_buffer = [queue commandBuffer];
520 [command_buffer addCompletedHandler:[callback](id<MTLCommandBuffer> commandBuffer) {
521#ifdef PROFILE_KERNELS
525#ifdef PROFILE_KERNELS
530 [command_buffer commit];
543 for (
auto &out : nodes) {
544 std::cout << static_cast<float *> ([kernel_arguments[out.get()] contents])[index] <<
" ";
546 std::cout << std::endl;
559 return static_cast<float *
> ([kernel_arguments[node.get()] contents])[index];
570 const size_t size = [kernel_arguments[node.get()] length];
571 memcpy([kernel_arguments[node.get()] contents],
582 float *destination) {
583 command_buffer = [queue commandBuffer];
585 [command_buffer commit];
586 [command_buffer waitUntilCompleted];
589 kernel_arguments[node.get()].contents,
590 kernel_arguments[node.get()].length);
599 source_buffer <<
"#include <metal_stdlib>" << std::endl;
600 source_buffer <<
"#include <metal_simdgroup>" << std::endl;
601 source_buffer <<
"using namespace metal;" << std::endl;
624 const std::string name,
630 const std::vector<bool> &is_constant,
637 const size_t iterations=1) {
638 source_buffer << std::endl;
639 source_buffer <<
"kernel void " << name <<
"(" << std::endl;
641 bufferMutability[name] = std::vector<MTLMutability> ();
643 size_t used_thread_mem = 0;
645 size_t buffer_count = 0;
647 for (
size_t i = 0, ie = inputs.size(); i < ie; i++) {
648 if (!used_args.contains(inputs[i].get())) {
649#ifdef USE_INPUT_CACHE
650 if (!is_constant[i] && iterations > 1) {
651 const size_t needed_mem = inputs[i]->size() > 1024 ?
654 if (used_thread_mem + needed_mem < device.maxThreadgroupMemoryLength) {
655 used_thread_mem += needed_mem;
656 thread_shared.insert(inputs[i].get());
658 }
else if (is_constant[i] &&
659 inputs[i]->size() < size &&
660 inputs[i]->size() < 1024) {
661 const size_t needed_mem = inputs[i]->size()*4;
662 if (used_thread_mem + needed_mem < device.maxThreadgroupMemoryLength) {
663 used_thread_mem += needed_mem;
664 thread_shared.insert(inputs[i].get());
665 thread_mem[inputs[i].get()] =
jit::to_string(
't', inputs[i].get());
669 bufferMutability[name].push_back(is_constant[i] ? MTLMutabilityMutable : MTLMutabilityImmutable);
670 source_buffer <<
" " << (is_constant[i] ?
"constant" :
"device")
673 <<
" [[buffer(" << buffer_count++ <<
")]]";
674 inputs[i]->endline(source_buffer, usage,
',');
675 used_args.insert(inputs[i].get());
679 for (
size_t i = 0, ie = outputs.size(); i < ie; i++) {
680 if (!used_args.contains(outputs[i].get()) &&
682 bufferMutability[name].push_back(MTLMutabilityMutable);
683 source_buffer <<
" device float *"
685 <<
" [[buffer(" << buffer_count++ <<
")]],"
687 used_args.insert(outputs[i].get());
691 for (
size_t i = 0, ie = atomics.size(); i < ie; i++) {
692 if (!used_args.contains(atomics[i].get())) {
693 bufferMutability[name].push_back(MTLMutabilityMutable);
694 source_buffer <<
" device atomic_float *"
696 <<
" [[buffer(" << buffer_count++ <<
")]],"
698 used_args.insert(atomics[i].get());
703 bufferMutability[name].push_back(MTLMutabilityMutable);
704 source_buffer <<
" device mt_state *"
706 <<
" [[buffer(" << buffer_count++ <<
")]],"
708 <<
" constant uint32_t &offset [[buffer("
709 << buffer_count++ <<
")]],"
713 for (
auto &[key, value] : textures1d) {
714 source_buffer <<
" const texture1d<float, access::read> "
716 <<
" [[texture(" << index++ <<
")]],"
719 for (
auto &[key, value] : textures2d) {
720 source_buffer <<
" const texture2d<float, access::read> "
722 <<
" [[texture(" << index++ <<
")]],"
725 if (thread_shared.size()) {
726 source_buffer <<
" ushort t_index [[thread_position_in_threadgroup]]," << std::endl;
727 source_buffer <<
" ushort t_total [[threads_per_threadgroup]]," << std::endl;
731 <<
" index [[thread_position_in_grid]]) {" << std::endl
734 source_buffer <<
"offset + ";
736 source_buffer <<
"index < " << size <<
") {" << std::endl;
738 for (
size_t i = 0, ie = inputs.size(); i < ie; i++) {
739 if (is_constant[i]) {
740#ifdef USE_INPUT_CACHE
741 if (usage.at(inputs[i].get()) && inputs[i]->size() == size) {
742 registers[inputs[i].get()] =
jit::to_string(
'r', inputs[i].get());
743 source_buffer <<
" const ";
745 source_buffer <<
" " << registers[inputs[i].get()] <<
" = "
749 source_buffer <<
"offset + ";
751 source_buffer <<
"index]";
752 inputs[i]->endline(source_buffer, usage);
757 + (state.get() ?
"offset + " :
"")
763 if (thread_shared.size()) {
764 for (
size_t i = 0, ie = inputs.size(); i < ie; i++) {
765 if (thread_shared.contains(inputs[i].get()) && is_constant[i]) {
766 source_buffer <<
" threadgroup float "
768 <<
"[" << inputs[i]->size() <<
"]";
769 inputs[i]->endline(source_buffer, usage);
772 for (
size_t i = 0, ie = inputs.size(); i < ie; i++) {
773 if (thread_shared.contains(inputs[i].get()) && is_constant[i]) {
774 source_buffer <<
" for(int j = t_index; j < "
776 <<
"; j += t_total) {" << std::endl
782 inputs[i]->endline(source_buffer, usage);
783 source_buffer <<
" }" << std::endl;
786 for (
size_t i = 0, ie = inputs.size(); i < ie; i++) {
787 if (thread_shared.contains(inputs[i].get()) && is_constant[i]) {
788 source_buffer <<
" threadgroup_barrier(mem_flags::mem_none);"
793 for (
size_t i = 0, ie = inputs.size(); i < ie; i++) {
794 if (thread_shared.contains(inputs[i].get()) && is_constant[i]) {
795 thread_shared.erase(inputs[i].get());
798 for (
size_t i = 0, ie = inputs.size(); i < ie; i++) {
799 if (thread_shared.contains(inputs[i].get()) && !is_constant[i]) {
800 source_buffer <<
" threadgroup float "
803 << (inputs[i]->size() > 1024 ? 1024 : 32)
805 inputs[i]->endline(source_buffer, usage);
812 source_buffer <<
"offset + ";
814 source_buffer <<
"index]";
815 inputs[i]->endline(source_buffer, usage);
820 if (iterations > 1) {
821 source_buffer <<
" for (size_t j = 0; j < " << iterations <<
"; j++) {" << std::endl;
824 for (
size_t i = 0, ie = inputs.size(); i < ie; i++) {
825 if (!is_constant[i]) {
826#ifdef USE_INPUT_CACHE
827 if (usage.at(inputs[i].get())) {
828 registers[inputs[i].get()] =
jit::to_string(
'r', inputs[i].get());
829 source_buffer <<
" const ";
831 source_buffer <<
" " << registers[inputs[i].get()] <<
" = ";
832 if (thread_shared.contains(inputs[i].get())) {
839 source_buffer <<
"offset + ";
841 source_buffer <<
"index]";
843 inputs[i]->endline(source_buffer, usage);
846 if (thread_shared.contains(inputs[i].get())) {
847 registers[inputs[i].get()] =
jit::to_string(
't', inputs[i].get()) +
"[t_index]";
849 registers[inputs[i].get()] =
jit::to_string(
'v', inputs[i].get()) +
"["
850 + (state.get() ?
"offset + " :
"")
857#ifdef USE_INPUT_CACHE
859 source_buffer <<
" device mt_state &" << registers[state.get()]
862 state->endline(source_buffer, usage);
891 const size_t iterations=1) {
893 for (
auto &[out, in] : setters) {
894 if (!out->is_match(in)) {
895 auto a = out->compile(source_buffer, registers,
897 source_buffer <<
" ";
898 if (thread_shared.contains(in.get())) {
905 source_buffer <<
"offset + ";
907 source_buffer <<
"index] = ";
909 if constexpr (SAFE_MATH) {
910 source_buffer <<
"isnan(" << registers[a.get()]
913 source_buffer << registers[a.get()] <<
";" << std::endl;
914 out_registers.insert(out.get());
918 for (
auto &out : outputs) {
921 !out_registers.contains(out.get())) {
922 auto a = out->compile(source_buffer, registers,
927 source_buffer <<
"offset + ";
929 source_buffer <<
"index] = ";
930 if constexpr (SAFE_MATH) {
932 source_buffer <<
"isnan(" << registers[a.get()]
936 source_buffer << registers[a.get()] <<
";" << std::endl;
937 out_registers.insert(out.get());
941 if (iterations > 1) {
942 source_buffer <<
" }" << std::endl;
944 for (
auto &[out, in] : setters) {
945 if (thread_shared.contains(in.get())) {
950 source_buffer <<
"offset + ";
952 source_buffer <<
"index] = "
954 <<
"[t_index];" << std::endl;
957 source_buffer <<
" }" << std::endl <<
"}" << std::endl;
968 source_buffer << std::endl;
969 source_buffer <<
"kernel void max_reduction(" << std::endl;
970 source_buffer <<
" constant float *input [[buffer(0)]]," << std::endl;
971 source_buffer <<
" device float *result [[buffer(1)]]," << std::endl;
972 source_buffer <<
" uint i [[thread_position_in_grid]]," << std::endl;
973 source_buffer <<
" uint j [[simdgroup_index_in_threadgroup]]," << std::endl;
974 source_buffer <<
" uint k [[thread_index_in_simdgroup]]) {" << std::endl;
975 source_buffer <<
" if (i < " << size <<
") {" << std::endl;
976 source_buffer <<
" float sub_max = input[i];" << std::endl;
977 source_buffer <<
" for (size_t index = i + 1024; index < " << size <<
"; index += 1024) {" << std::endl;
978 source_buffer <<
" sub_max = max(sub_max, input[index]);" << std::endl;
979 source_buffer <<
" }" << std::endl;
980 source_buffer <<
" threadgroup float thread_max[32];" << std::endl;
981 source_buffer <<
" thread_max[j] = simd_max(sub_max);" << std::endl;
982 source_buffer <<
" threadgroup_barrier(mem_flags::mem_none);" << std::endl;
983 source_buffer <<
" if (j == 0) {" << std::endl;
984 source_buffer <<
" *result = simd_max(thread_max[k]);" << std::endl;
985 source_buffer <<
" }" << std::endl;
986 source_buffer <<
" }" << std::endl;
987 source_buffer <<
"}" << std::endl << std::endl;
996 return static_cast<float *
> ([kernel_arguments[node.get()] contents]);
Class representing a generic buffer.
Definition backend.hpp:29
std::function< void(void)> create_copy_call(graph::copy_nodes< float, SAFE_MATH > &setters)
Create kernel call that will to copy one buffer to another.
Definition metal_context.hpp:447
void create_reduction(std::ostringstream &source_buffer, const size_t size)
Create reduction.
Definition metal_context.hpp:966
void create_kernel_postfix(std::ostringstream &source_buffer, graph::output_nodes< float, SAFE_MATH > &outputs, graph::map_nodes< float, SAFE_MATH > &setters, graph::shared_random_state< float, SAFE_MATH > state, jit::register_map ®isters, const jit::register_usage &usage, const jit::argument_set &thread_shared, jit::register_map &thread_mem, const size_t iterations=1)
Create kernel postfix.
Definition metal_context.hpp:883
static size_t max_concurrency()
Get the maximum number of concurrent instances.
Definition metal_context.hpp:56
MTLCompileOptions * compile_options()
Get the compile options.
Definition metal_context.hpp:491
void wait()
Hold the current thread until the command buffer has completed.
Definition metal_context.hpp:503
std::function< void(void)> run_function(std::function< void(void)> callback)
Run a callback function in the queue.
Definition metal_context.hpp:516
float check_value(const size_t index, const graph::shared_leaf< float, SAFE_MATH > &node)
Check the value.
Definition metal_context.hpp:556
void create_kernel_prefix(std::ostringstream &source_buffer, const std::string name, graph::input_nodes< float, SAFE_MATH > &inputs, graph::output_nodes< float, SAFE_MATH > &outputs, graph::atomic_nodes< float, SAFE_MATH > atomics, graph::shared_random_state< float, SAFE_MATH > &state, const size_t size, const std::vector< bool > &is_constant, jit::register_map ®isters, const jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, jit::argument_set &thread_shared, jit::register_map &thread_mem, const size_t iterations=1)
Create kernel prefix.
Definition metal_context.hpp:623
void compile(const std::string kernel_source, std::vector< std::string > names, const bool add_reduction=false)
Compile the kernels.
Definition metal_context.hpp:83
void print_results(const size_t index, const graph::output_nodes< float, SAFE_MATH > &nodes)
Print out the results.
Definition metal_context.hpp:540
std::function< void(void)> create_kernel_call(const std::string kernel_name, graph::input_nodes< float, SAFE_MATH > inputs, graph::output_nodes< float, SAFE_MATH > outputs, graph::atomic_nodes< float, SAFE_MATH > atomics, graph::shared_random_state< float, SAFE_MATH > state, const size_t num_rays, const jit::texture1d_list &tex1d_list, const jit::texture2d_list &tex2d_list)
Create a kernel calling function.
Definition metal_context.hpp:120
float * get_buffer(graph::shared_leaf< float, SAFE_MATH > &node)
Get the buffer for a node.
Definition metal_context.hpp:995
static std::string device_type()
Device description.
Definition metal_context.hpp:63
void copy_to_device(graph::shared_leaf< float, SAFE_MATH > node, float *source)
Copy buffer contents to the device.
Definition metal_context.hpp:568
metal_context(const size_t index)
Construct a metal context.
Definition metal_context.hpp:72
static constexpr size_t random_state_size
Size of random state needed.
Definition metal_context.hpp:46
int remaining_const_memory
Remaining constant memory in bytes. NOT USED.
Definition metal_context.hpp:49
static constexpr size_t random_state_scale
Random state size multiplier.
Definition metal_context.hpp:44
std::function< float(void)> create_max_call(graph::shared_leaf< float, SAFE_MATH > &argument, std::function< void(void)> run)
Create a max compute kernel calling function.
Definition metal_context.hpp:340
void copy_to_host(graph::shared_leaf< float, SAFE_MATH > node, float *destination)
Copy buffer contents to host.
Definition metal_context.hpp:581
std::function< void(void)> create_zero_call(graph::input_nodes< float, SAFE_MATH > &inputs)
Create kernel call that will be memset a buffer to zero.
Definition metal_context.hpp:406
void create_header(std::ostringstream &source_buffer)
Create the source header.
Definition metal_context.hpp:598
A timing object.
Definition timing.hpp:18
void print() const
Print the result.
Definition timing.hpp:52
Name space for GPU backends.
Definition cpu_context.hpp:52
std::vector< shared_leaf< T, SAFE_MATH > > output_nodes
Convenience type alias for a vector of output nodes.
Definition node.hpp:705
std::vector< std::pair< shared_leaf< T, SAFE_MATH >, shared_variable< T, SAFE_MATH > > > map_nodes
Convenience type alias for mapping end codes back to inputs.
Definition node.hpp:2151
std::vector< std::pair< shared_variable< T, SAFE_MATH >, shared_variable< T, SAFE_MATH > > > copy_nodes
Convenience type alias for copying buffers.
Definition node.hpp:2155
shared_atomic_accumulate_1D< T, SAFE_MATH > atomic_accumulate_1D_cast(shared_leaf< T, SAFE_MATH > x)
Cast to a atomic accumulate 1D node.
Definition piecewise.hpp:2426
std::shared_ptr< leaf_node< T, SAFE_MATH > > shared_leaf
Convenience type alias for shared leaf nodes.
Definition node.hpp:690
input_nodes< T, SAFE_MATH > atomic_nodes
Convenience type alias for atomic inputd.
Definition node.hpp:2159
shared_variable< T, SAFE_MATH > variable_cast(shared_leaf< T, SAFE_MATH > x)
Cast to a variable node.
Definition node.hpp:2171
std::shared_ptr< random_state_node< T, SAFE_MATH > > shared_random_state
Convenience type alias for shared sqrt nodes.
Definition random.hpp:269
std::shared_ptr< variable_node< T, SAFE_MATH > > shared_variable
Convenience type alias for shared variable nodes.
Definition node.hpp:2145
shared_random< T, SAFE_MATH > random_cast(shared_leaf< T, SAFE_MATH > x)
Cast to a random node.
Definition random.hpp:539
std::vector< shared_variable< T, SAFE_MATH > > input_nodes
Convenience type alias for a vector of inputs.
Definition node.hpp:2148
std::unordered_map< void *, size_t > register_usage
Type alias for counting register usage.
Definition register.hpp:261
void add_type(std::basic_ostream< char > &stream)
Write out the node base type to a general stream.
Definition register.hpp:165
std::string smallest_uint_type(const size_t max_size)
Get smallest integer type.
Definition register.hpp:109
std::unordered_map< void *, std::string > register_map
Type alias for mapping node pointers to register names.
Definition register.hpp:259
std::unordered_map< void *, std::array< size_t, 2 > > texture2d_list
Type alias for indexing 2D textures.
Definition register.hpp:267
std::unordered_map< void *, size_t > texture1d_list
Type alias for indexing 1D textures.
Definition register.hpp:265
std::string to_string(const char prefix, const NODE *pointer)
Convert a graph::leaf_node pointer to a string.
Definition register.hpp:247
std::unordered_set< void * > argument_set
Type for tacking thread shared memory.
Definition register.hpp:271
Name space for output files.
Definition output.hpp:16
Random constants and distributions.
Routines to time the ray execution.