Graph Framework
Loading...
Searching...
No Matches
metal_context.hpp
Go to the documentation of this file.
1//------------------------------------------------------------------------------
6//------------------------------------------------------------------------------
7
8#ifndef metal_context_h
9#define metal_context_h
10
11#import <Metal/Metal.h>
12
13#include "random.hpp"
14#include "timing.hpp"
15#include "piecewise.hpp"
16
18namespace gpu {
19//------------------------------------------------------------------------------
23//------------------------------------------------------------------------------
24 template<bool SAFE_MATH=false>
26 private:
28 id<MTLDevice> device;
30 id<MTLCommandQueue> queue;
32 std::unordered_map<graph::leaf_node<float, SAFE_MATH> *, id<MTLBuffer>> kernel_arguments;
34 std::unordered_map<void *, id<MTLTexture>> texture_arguments;
36 id<MTLCommandBuffer> command_buffer;
38 id<MTLLibrary> library;
40 std::unordered_map<std::string, std::vector<MTLMutability>> bufferMutability;
41
42 public:
44 constexpr static size_t random_state_scale = 3000;
46 constexpr static size_t random_state_size = 1024*random_state_scale;
47
50
51//------------------------------------------------------------------------------
55//------------------------------------------------------------------------------
56 static size_t max_concurrency() {
57 return MTLCopyAllDevices().count;
58 }
59
60//------------------------------------------------------------------------------
62//------------------------------------------------------------------------------
63 static std::string device_type() {
64 return "Metal GPU";
65 }
66
67//------------------------------------------------------------------------------
71//------------------------------------------------------------------------------
72 metal_context(const size_t index) :
73 device([MTLCopyAllDevices() objectAtIndex:index]),
74 queue([device newCommandQueue]) {}
75
76//------------------------------------------------------------------------------
82//------------------------------------------------------------------------------
83 void compile(const std::string kernel_source,
84 std::vector<std::string> names,
85 const bool add_reduction=false) {
86 NSError *error;
87 library = [device newLibraryWithSource:[NSString stringWithCString:kernel_source.c_str()
88 encoding:NSUTF8StringEncoding]
89 options:compile_options()
90 error:&error];
91
92 if (error) {
93 NSLog(@"%@", error);
94 }
95
96 if (jit::verbose) {
97 std::cout << "Metal GPU info." << std::endl
98 << " Max thread group memory : " << device.maxThreadgroupMemoryLength << std::endl
99 << " Max thread per group : " << device.maxThreadsPerThreadgroup.width << std::endl
100 << " Device name : " << [device.name cStringUsingEncoding:NSString.defaultCStringEncoding] << std::endl
101 << " Architecture : " << [device.architecture.name cStringUsingEncoding:NSString.defaultCStringEncoding] << std::endl
102 << " Max buffer length : " << device.maxBufferLength << std::endl
103 << " Max working set : " << device.recommendedMaxWorkingSetSize << std::endl;
104 }
105 }
106
107//------------------------------------------------------------------------------
119//------------------------------------------------------------------------------
120 std::function<void(void)> create_kernel_call(const std::string kernel_name,
125 const size_t num_rays,
126 const jit::texture1d_list &tex1d_list,
127 const jit::texture2d_list &tex2d_list) {
128 NSError *error;
129
130 MTLFunctionDescriptor *funcDesc = [MTLFunctionDescriptor new];
131 funcDesc.options = MTLFunctionOptionNone;
132 funcDesc.name = [NSString stringWithCString:kernel_name.c_str()
133 encoding:NSUTF8StringEncoding];
134
135 id<MTLFunction> function = [library newFunctionWithDescriptor:funcDesc
136 error:&error];
137
138 MTLComputePipelineDescriptor *compute = [MTLComputePipelineDescriptor new];
139 compute.threadGroupSizeIsMultipleOfThreadExecutionWidth = YES;
140 compute.computeFunction = function;
141 compute.maxTotalThreadsPerThreadgroup = 1024;
142 for (size_t i = 0, ie = bufferMutability[kernel_name].size(); i < ie; i++) {
143 compute.buffers[i].mutability = bufferMutability[kernel_name][i];
144 }
145
146 id<MTLComputePipelineState> pipeline = [device newComputePipelineStateWithDescriptor:compute
147 options:MTLPipelineOptionNone
148 reflection:NULL
149 error:&error];
150
151 if (error) {
152 NSLog(@"%@", error);
153 }
154
155 std::vector<id<MTLBuffer>> buffers;
156 std::unordered_set<graph::leaf_node<float, SAFE_MATH> *> needed_buffers;
157
158 const size_t buffer_element_size = sizeof(float);
159 for (graph::shared_variable<float, SAFE_MATH> &input : inputs) {
160 if (!kernel_arguments.contains(input.get())) {
161 backend::buffer<float> buffer = input->evaluate();
162 kernel_arguments[input.get()] = [device newBufferWithBytes:buffer.data()
163 length:buffer.size()*buffer_element_size
164 options:MTLResourceStorageModeShared];
165 }
166 if (!needed_buffers.contains(input.get())) {
167 buffers.push_back(kernel_arguments[input.get()]);
168 needed_buffers.insert(input.get());
169 }
170 }
173 if (!kernel_arguments.contains(output.get())) {
174 kernel_arguments[output.get()] = [device newBufferWithLength:num_rays*sizeof(float)
175 options:MTLResourceStorageModeShared];
176 }
177 if (!needed_buffers.contains(output.get())) {
178 buffers.push_back(kernel_arguments[output.get()]);
179 needed_buffers.insert(output.get());
180 }
181 }
182 }
183 for (graph::shared_variable<float, SAFE_MATH> &atomic : atomics) {
184 if (!kernel_arguments.contains(atomic.get())) {
185 kernel_arguments[atomic.get()] = [device newBufferWithLength:atomic->size()*buffer_element_size
186 options:MTLResourceStorageModeShared];
187 }
188 if (!needed_buffers.contains(atomic.get())) {
189 buffers.push_back(kernel_arguments[atomic.get()]);
190 needed_buffers.insert(atomic.get());
191 }
192 }
193 if (state.get()) {
194 if (!kernel_arguments.contains(state.get())) {
195 kernel_arguments[state.get()] = [device newBufferWithBytes:state->data()
196 length:state->get_size_bytes()
197 options:MTLResourceCPUCacheModeWriteCombined |
198 MTLResourceStorageModeShared |
199 MTLResourceHazardTrackingModeUntracked];
200 }
201 buffers.push_back(kernel_arguments[state.get()]);
202 }
203
204 std::vector<id<MTLTexture>> textures;
205 command_buffer = [queue commandBuffer];
206 id<MTLBlitCommandEncoder> encoder = [command_buffer blitCommandEncoder];
207 for (auto &[data, size] : tex1d_list) {
208 if (!texture_arguments.contains(data)) {
209 MTLTextureDescriptor *descriptor = [MTLTextureDescriptor new];
210 descriptor.textureType = MTLTextureType1D;
211 descriptor.pixelFormat = MTLPixelFormatR32Float;
212 descriptor.width = size;
213 descriptor.storageMode = MTLStorageModeShared;
214 descriptor.cpuCacheMode = MTLCPUCacheModeWriteCombined;
215 descriptor.hazardTrackingMode = MTLHazardTrackingModeUntracked;
216 descriptor.usage = MTLTextureUsageShaderRead;
217 texture_arguments[data] = [device newTextureWithDescriptor:descriptor];
218 [texture_arguments[data] replaceRegion:MTLRegionMake1D(0, size)
219 mipmapLevel:0
220 withBytes:reinterpret_cast<float *> (data)
221 bytesPerRow:4*size];
222
223 [encoder optimizeContentsForGPUAccess:texture_arguments[data]];
224 }
225 textures.push_back(texture_arguments[data]);
226 }
227 for (auto &[data, size] : tex2d_list) {
228 if (!texture_arguments.contains(data)) {
229 MTLTextureDescriptor *descriptor = [MTLTextureDescriptor new];
230 descriptor.textureType = MTLTextureType2D;
231 descriptor.pixelFormat = MTLPixelFormatR32Float;
232 descriptor.width = size[1];
233 descriptor.height = size[0];
234 descriptor.storageMode = MTLStorageModeShared;
235 descriptor.cpuCacheMode = MTLCPUCacheModeWriteCombined;
236 descriptor.hazardTrackingMode = MTLHazardTrackingModeUntracked;
237 descriptor.usage = MTLTextureUsageShaderRead;
238 texture_arguments[data] = [device newTextureWithDescriptor:descriptor];
239 [texture_arguments[data] replaceRegion:MTLRegionMake2D(0, 0, size[1], size[0])
240 mipmapLevel:0
241 withBytes:reinterpret_cast<float *> (data)
242 bytesPerRow:4*size[1]];
243
244 [encoder optimizeContentsForGPUAccess:texture_arguments[data]];
245 }
246 textures.push_back(texture_arguments[data]);
247 }
248 [encoder endEncoding];
249 [command_buffer commit];
250
251 std::vector<NSUInteger> offsets(buffers.size(), 0);
252 NSRange range = NSMakeRange(0, buffers.size());
253 NSRange tex_range = NSMakeRange(0, textures.size());
254
255 NSUInteger total_parallel = state.get() ? state->size() : num_rays;
256 NSUInteger thread_width = pipeline.threadExecutionWidth;
257 NSUInteger threads_per_group = total_parallel < pipeline.maxTotalThreadsPerThreadgroup ?
258 thread_width :
259 pipeline.maxTotalThreadsPerThreadgroup;
260 NSUInteger thread_groups = total_parallel/threads_per_group + (total_parallel%threads_per_group ? 1 : 0);
261
262 if (jit::verbose) {
263 std::cout << " Kernel name : " << kernel_name << std::endl
264 << " Thread execution width : " << thread_width << std::endl
265 << " Threads per group : " << threads_per_group << std::endl
266 << " Number of groups : " << thread_groups << std::endl
267 << " Total problem size : " << threads_per_group*thread_groups << std::endl
268 << " Total parallel size : " << total_parallel << std::endl
269 << " Current allocation size : " << device.currentAllocatedSize << std::endl;
270 }
271
272 if (state.get()) {
273
274 return [this, num_rays, pipeline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures
275#ifdef PROFILE_KERNELS
276 , kernel_name
277#endif
278 ] () mutable {
279 command_buffer = [queue commandBuffer];
280 for (NSUInteger i = 0, ie = thread_groups*threads_per_group; i < num_rays; i += ie) {
281 id<MTLComputeCommandEncoder> encoder = [command_buffer computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial];
282
283 [encoder setComputePipelineState:pipeline];
284 [encoder setBuffers:buffers.data()
285 offsets:offsets.data()
286 withRange:range];
287 [encoder setBytes:&i
288 length:sizeof(uint32_t)
289 atIndex:buffers.size()];
290 [encoder setTextures:textures.data()
291 withRange:tex_range];
292
293 [encoder dispatchThreadgroups:MTLSizeMake(thread_groups, 1, 1)
294 threadsPerThreadgroup:MTLSizeMake(threads_per_group, 1, 1)];
295 [encoder endEncoding];
296 }
297#ifdef PROFILE_KERNELS
298 [command_buffer addCompletedHandler:[kernel_name](id<MTLCommandBuffer> commandBuffer) {
299 std::cout << std::endl << " " << kernel_name << " : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime << " s" << std::endl << std::endl;
300 }];
301#endif
302 [command_buffer commit];
303 };
304 } else {
305 return [this, pipeline, buffers, offsets, range, tex_range, thread_groups, threads_per_group, textures
306#ifdef PROFILE_KERNELS
307 , kernel_name
308#endif
309 ] () mutable {
310 command_buffer = [queue commandBuffer];
311 id<MTLComputeCommandEncoder> encoder = [command_buffer computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial];
312
313 [encoder setComputePipelineState:pipeline];
314 [encoder setBuffers:buffers.data()
315 offsets:offsets.data()
316 withRange:range];
317 [encoder setTextures:textures.data()
318 withRange:tex_range];
319
320 [encoder dispatchThreadgroups:MTLSizeMake(thread_groups, 1, 1)
321 threadsPerThreadgroup:MTLSizeMake(threads_per_group, 1, 1)];
322 [encoder endEncoding];
323#ifdef PROFILE_KERNELS
324 [command_buffer addCompletedHandler:[kernel_name](id<MTLCommandBuffer> commandBuffer) {
325 std::cout << std::endl << " " << kernel_name << " : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime << " s" << std::endl << std::endl;
326 }];
327#endif
328 [command_buffer commit];
329 };
330 }
331 }
332
333//------------------------------------------------------------------------------
339//------------------------------------------------------------------------------
340 std::function<float(void)> create_max_call(graph::shared_leaf<float, SAFE_MATH> &argument,
341 std::function<void(void)> run) {
342 NSError *error;
343
344 MTLFunctionDescriptor *funcDesc = [MTLFunctionDescriptor new];
345 funcDesc.options = MTLFunctionOptionNone;
346 funcDesc.name = @"max_reduction";
347
348 id<MTLFunction> function = [library newFunctionWithDescriptor:funcDesc
349 error:&error];
350
351 MTLComputePipelineDescriptor *compute = [MTLComputePipelineDescriptor new];
352 compute.threadGroupSizeIsMultipleOfThreadExecutionWidth = YES;
353 compute.computeFunction = function;
354 compute.maxTotalThreadsPerThreadgroup = 1024;
355 compute.buffers[0].mutability = MTLMutabilityImmutable;
356
357 id<MTLComputePipelineState> max_state = [device newComputePipelineStateWithDescriptor:compute
358 options:MTLPipelineOptionNone
359 reflection:NULL
360 error:&error];
361 if (error) {
362 NSLog(@"%@", error);
363 }
364
365 id<MTLBuffer> result = [device newBufferWithLength:sizeof(float)
366 options:MTLResourceStorageModeShared];
367
368 id<MTLBuffer> buffer = kernel_arguments[argument.get()];
369
370 NSUInteger threads_per_group = max_state.maxTotalThreadsPerThreadgroup;
371 NSUInteger thread_width = max_state.threadExecutionWidth;
372 if (jit::verbose) {
373 std::cout << " Kernel name : max_reduction" << std::endl;
374 std::cout << " Thread execution width : " << thread_width << std::endl;
375 std::cout << " Threads per group : " << threads_per_group << std::endl;
376 std::cout << " Number of groups : " << 1 << std::endl;
377 std::cout << " Total problem size : " << threads_per_group*1 << std::endl;
378 }
379
380 return [this, run, buffer, result, max_state] () mutable {
381 run();
382 command_buffer = [queue commandBuffer];
383
384 id<MTLComputeCommandEncoder> encoder = [command_buffer computeCommandEncoderWithDispatchType:MTLDispatchTypeSerial];
385
386 [encoder setComputePipelineState:max_state];
387 [encoder setBuffer:buffer offset:0 atIndex:0];
388 [encoder setBuffer:result offset:0 atIndex:1];
389 [encoder dispatchThreadgroups:MTLSizeMake(1, 1, 1)
390 threadsPerThreadgroup:MTLSizeMake(1024, 1, 1)];
391 [encoder endEncoding];
392
393 [command_buffer commit];
394 [command_buffer waitUntilCompleted];
395
396 return static_cast<float *> (result.contents)[0];
397 };
398 }
399
400//------------------------------------------------------------------------------
405//------------------------------------------------------------------------------
406 std::function<void(void)> create_zero_call(graph::input_nodes<float, SAFE_MATH> &inputs) {
407 std::vector<id<MTLBuffer>> buffers;
408 for (auto &input : inputs) {
409 if (!kernel_arguments.contains(input.get())) {
410 kernel_arguments[input.get()] = [device newBufferWithBytes:input->data()
411 length:input->size()*sizeof(float)
412 options:MTLResourceStorageModeShared];
413 }
414 buffers.push_back(kernel_arguments[input.get()]);
415 }
416
417 std::vector<NSRange> ranges;
418 for (id<MTLBuffer> buffer : buffers) {
419 ranges.push_back(NSMakeRange(0, buffer.length));
420 }
421
422 return [this, buffers, ranges] () mutable {
423 command_buffer = [queue commandBuffer];
424 id<MTLBlitCommandEncoder> encoder = [command_buffer blitCommandEncoder];
425
426 for (size_t i = 0, ie = buffers.size(); i < ie; i++) {
427 [encoder fillBuffer:buffers[i]
428 range:ranges[i]
429 value:0];
430 }
431 [encoder endEncoding];
432#ifdef PROFILE_KERNELS
433 [command_buffer addCompletedHandler:[](id<MTLCommandBuffer> commandBuffer) {
434 std::cout << std::endl << " zero buffer : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime << " s" << std::endl << std::endl;
435 }];
436#endif
437 [command_buffer commit];
438 };
439 }
440
441//------------------------------------------------------------------------------
446//------------------------------------------------------------------------------
447 std::function<void(void)> create_copy_call(graph::copy_nodes<float, SAFE_MATH> &setters) {
448 std::vector<id<MTLBuffer>> sources;
449 std::vector<id<MTLBuffer>> destinations;
450
451 for (auto &[out, in] : setters) {
452 if (!kernel_arguments.contains(in.get())) {
453 kernel_arguments[in.get()] = [device newBufferWithBytes:in->data()
454 length:in->size()*sizeof(float)
455 options:MTLResourceStorageModeShared];
456 }
457 destinations.push_back(kernel_arguments[in.get()]);
458
459 if (!kernel_arguments.contains(out.get())) {
460 kernel_arguments[out.get()] = [device newBufferWithBytes:out->data()
461 length:out->size()*sizeof(float)
462 options:MTLResourceStorageModeShared];
463 }
464 sources.push_back(kernel_arguments[out.get()]);
465 }
466
467 return [this, sources, destinations] () mutable {
468 command_buffer = [queue commandBuffer];
469 id<MTLBlitCommandEncoder> encoder = [command_buffer blitCommandEncoder];
470
471 for (size_t i = 0, ie = sources.size(); i < ie; i++) {
472 [encoder copyFromBuffer:sources[i]
473 sourceOffset:0
474 toBuffer:destinations[i]
475 destinationOffset:0
476 size:sources[i].length];
477 }
478 [encoder endEncoding];
479#ifdef PROFILE_KERNELS
480 [command_buffer addCompletedHandler:[](id<MTLCommandBuffer> commandBuffer) {
481 std::cout << std::endl << " copy buffer : " << commandBuffer.GPUEndTime - commandBuffer.GPUStartTime << " s" << std::endl << std::endl;
482 }];
483#endif
484 [command_buffer commit];
485 };
486 }
487
488//------------------------------------------------------------------------------
490//------------------------------------------------------------------------------
491 MTLCompileOptions *compile_options() {
492 MTLCompileOptions *options = [MTLCompileOptions new];
493 options.mathMode = MTLMathModeFast;
494 options.mathFloatingPointFunctions = MTLMathFloatingPointFunctionsFast;
495 options.optimizationLevel = MTLLibraryOptimizationLevelDefault;
496 options.languageVersion = MTLLanguageVersion3_2;
497 return options;
498 }
499
500//------------------------------------------------------------------------------
502//------------------------------------------------------------------------------
503 void wait() {
504 command_buffer = [queue commandBuffer];
505
506 [command_buffer commit];
507 [command_buffer waitUntilCompleted];
508 }
509
510//------------------------------------------------------------------------------
515//------------------------------------------------------------------------------
516 std::function<void(void)> run_function(std::function<void(void)> callback) {
517 return [this, callback]() {
518 command_buffer = [queue commandBuffer];
519
520 [command_buffer addCompletedHandler:[callback](id<MTLCommandBuffer> commandBuffer) {
521#ifdef PROFILE_KERNELS
522 timing::measure_diagnostic timer("callback");
523#endif
524 callback();
525#ifdef PROFILE_KERNELS
526 timer.print();
527#endif
528 }];
529
530 [command_buffer commit];
531 };
532 }
533
534//------------------------------------------------------------------------------
539//------------------------------------------------------------------------------
540 void print_results(const size_t index,
542 wait();
543 for (auto &out : nodes) {
544 std::cout << static_cast<float *> ([kernel_arguments[out.get()] contents])[index] << " ";
545 }
546 std::cout << std::endl;
547 }
548
549//------------------------------------------------------------------------------
555//------------------------------------------------------------------------------
556 float check_value(const size_t index,
558 wait();
559 return static_cast<float *> ([kernel_arguments[node.get()] contents])[index];
560 }
561
562//------------------------------------------------------------------------------
567//------------------------------------------------------------------------------
569 float *source) {
570 const size_t size = [kernel_arguments[node.get()] length];
571 memcpy([kernel_arguments[node.get()] contents],
572 source, size);
573 }
574
575//------------------------------------------------------------------------------
580//------------------------------------------------------------------------------
582 float *destination) {
583 command_buffer = [queue commandBuffer];
584
585 [command_buffer commit];
586 [command_buffer waitUntilCompleted];
587
588 memcpy(destination,
589 kernel_arguments[node.get()].contents,
590 kernel_arguments[node.get()].length);
591 }
592
593//------------------------------------------------------------------------------
597//------------------------------------------------------------------------------
598 void create_header(std::ostringstream &source_buffer) {
599 source_buffer << "#include <metal_stdlib>" << std::endl;
600 source_buffer << "#include <metal_simdgroup>" << std::endl;
601 source_buffer << "using namespace metal;" << std::endl;
602 }
603
604//------------------------------------------------------------------------------
622//------------------------------------------------------------------------------
623 void create_kernel_prefix(std::ostringstream &source_buffer,
624 const std::string name,
629 const size_t size,
630 const std::vector<bool> &is_constant,
631 jit::register_map &registers,
632 const jit::register_usage &usage,
633 jit::texture1d_list &textures1d,
634 jit::texture2d_list &textures2d,
635 jit::argument_set &thread_shared,
636 jit::register_map &thread_mem,
637 const size_t iterations=1) {
638 source_buffer << std::endl;
639 source_buffer << "kernel void " << name << "(" << std::endl;
640
641 bufferMutability[name] = std::vector<MTLMutability> ();
642
643 size_t used_thread_mem = 0;
644
645 size_t buffer_count = 0;
646 jit::argument_set used_args;
647 for (size_t i = 0, ie = inputs.size(); i < ie; i++) {
648 if (!used_args.contains(inputs[i].get())) {
649#ifdef USE_INPUT_CACHE
650 if (!is_constant[i] && iterations > 1) {
651 const size_t needed_mem = inputs[i]->size() > 1024 ?
652 1024*4 :
653 32*4;
654 if (used_thread_mem + needed_mem < device.maxThreadgroupMemoryLength) {
655 used_thread_mem += needed_mem;
656 thread_shared.insert(inputs[i].get());
657 }
658 } else if (is_constant[i] &&
659 inputs[i]->size() < size &&
660 inputs[i]->size() < 1024) {
661 const size_t needed_mem = inputs[i]->size()*4;
662 if (used_thread_mem + needed_mem < device.maxThreadgroupMemoryLength) {
663 used_thread_mem += needed_mem;
664 thread_shared.insert(inputs[i].get());
665 thread_mem[inputs[i].get()] = jit::to_string('t', inputs[i].get());
666 }
667 }
668#endif
669 bufferMutability[name].push_back(is_constant[i] ? MTLMutabilityMutable : MTLMutabilityImmutable);
670 source_buffer << " " << (is_constant[i] ? "constant" : "device")
671 << " float *"
672 << jit::to_string('v', inputs[i].get())
673 << " [[buffer(" << buffer_count++ << ")]]";
674 inputs[i]->endline(source_buffer, usage, ',');
675 used_args.insert(inputs[i].get());
676 }
677 }
678
679 for (size_t i = 0, ie = outputs.size(); i < ie; i++) {
680 if (!used_args.contains(outputs[i].get()) &&
681 !graph::atomic_accumulate_1D_cast(outputs[i]).get()) {
682 bufferMutability[name].push_back(MTLMutabilityMutable);
683 source_buffer << " device float *"
684 << jit::to_string('o', outputs[i].get())
685 << " [[buffer(" << buffer_count++ << ")]],"
686 << std::endl;
687 used_args.insert(outputs[i].get());
688 }
689 }
690
691 for (size_t i = 0, ie = atomics.size(); i < ie; i++) {
692 if (!used_args.contains(atomics[i].get())) {
693 bufferMutability[name].push_back(MTLMutabilityMutable);
694 source_buffer << " device atomic_float *"
695 << jit::to_string('v', atomics[i].get())
696 << " [[buffer(" << buffer_count++ << ")]],"
697 << std::endl;
698 used_args.insert(atomics[i].get());
699 }
700 }
701
702 if (state.get()) {
703 bufferMutability[name].push_back(MTLMutabilityMutable);
704 source_buffer << " device mt_state *"
705 << jit::to_string('s', state.get())
706 << " [[buffer(" << buffer_count++ << ")]],"
707 << std::endl
708 << " constant uint32_t &offset [[buffer("
709 << buffer_count++ << ")]],"
710 << std::endl;
711 }
712 size_t index = 0;
713 for (auto &[key, value] : textures1d) {
714 source_buffer << " const texture1d<float, access::read> "
715 << jit::to_string('a', key)
716 << " [[texture(" << index++ << ")]],"
717 << std::endl;
718 }
719 for (auto &[key, value] : textures2d) {
720 source_buffer << " const texture2d<float, access::read> "
721 << jit::to_string('a', key)
722 << " [[texture(" << index++ << ")]],"
723 << std::endl;
724 }
725 if (thread_shared.size()) {
726 source_buffer << " ushort t_index [[thread_position_in_threadgroup]]," << std::endl;
727 source_buffer << " ushort t_total [[threads_per_threadgroup]]," << std::endl;
728 }
729 source_buffer << " "
731 << " index [[thread_position_in_grid]]) {" << std::endl
732 << " if (";
733 if (state.get()) {
734 source_buffer << "offset + ";
735 }
736 source_buffer << "index < " << size << ") {" << std::endl;
737
738 for (size_t i = 0, ie = inputs.size(); i < ie; i++) {
739 if (is_constant[i]) {
740#ifdef USE_INPUT_CACHE
741 if (usage.at(inputs[i].get()) && inputs[i]->size() == size) {
742 registers[inputs[i].get()] = jit::to_string('r', inputs[i].get());
743 source_buffer << " const ";
744 jit::add_type<float> (source_buffer);
745 source_buffer << " " << registers[inputs[i].get()] << " = "
746 << jit::to_string('v', inputs[i].get())
747 << "[";
748 if (state.get()) {
749 source_buffer << "offset + ";
750 }
751 source_buffer << "index]";
752 inputs[i]->endline(source_buffer, usage);
753 }
754#else
755 registers[inputs[i].get()] = jit::to_string('v', inputs[i].get())
756 + "["
757 + (state.get() ? "offset + " : "")
758 + "index]";
759#endif
760 }
761 }
762
763 if (thread_shared.size()) {
764 for (size_t i = 0, ie = inputs.size(); i < ie; i++) {
765 if (thread_shared.contains(inputs[i].get()) && is_constant[i]) {
766 source_buffer << " threadgroup float "
767 << jit::to_string('t', inputs[i].get())
768 << "[" << inputs[i]->size() << "]";
769 inputs[i]->endline(source_buffer, usage);
770 }
771 }
772 for (size_t i = 0, ie = inputs.size(); i < ie; i++) {
773 if (thread_shared.contains(inputs[i].get()) && is_constant[i]) {
774 source_buffer << " for(int j = t_index; j < "
775 << inputs[i]->size()
776 << "; j += t_total) {" << std::endl
777 << " "
778 << jit::to_string('t', inputs[i].get())
779 << "[j] = "
780 << jit::to_string('v', inputs[i].get())
781 << "[j]";
782 inputs[i]->endline(source_buffer, usage);
783 source_buffer << " }" << std::endl;
784 }
785 }
786 for (size_t i = 0, ie = inputs.size(); i < ie; i++) {
787 if (thread_shared.contains(inputs[i].get()) && is_constant[i]) {
788 source_buffer << " threadgroup_barrier(mem_flags::mem_none);"
789 << std::endl;
790 break;
791 }
792 }
793 for (size_t i = 0, ie = inputs.size(); i < ie; i++) {
794 if (thread_shared.contains(inputs[i].get()) && is_constant[i]) {
795 thread_shared.erase(inputs[i].get());
796 }
797 }
798 for (size_t i = 0, ie = inputs.size(); i < ie; i++) {
799 if (thread_shared.contains(inputs[i].get()) && !is_constant[i]) {
800 source_buffer << " threadgroup float "
801 << jit::to_string('t', inputs[i].get())
802 << "["
803 << (inputs[i]->size() > 1024 ? 1024 : 32)
804 << "]";
805 inputs[i]->endline(source_buffer, usage);
806 source_buffer << " "
807 << jit::to_string('t', inputs[i].get())
808 << "[t_index] = "
809 << jit::to_string('v', inputs[i].get())
810 << "[";
811 if (state.get()) {
812 source_buffer << "offset + ";
813 }
814 source_buffer << "index]";
815 inputs[i]->endline(source_buffer, usage);
816 }
817 }
818 }
819
820 if (iterations > 1) {
821 source_buffer << " for (size_t j = 0; j < " << iterations << "; j++) {" << std::endl;
822 }
823
824 for (size_t i = 0, ie = inputs.size(); i < ie; i++) {
825 if (!is_constant[i]) {
826#ifdef USE_INPUT_CACHE
827 if (usage.at(inputs[i].get())) {
828 registers[inputs[i].get()] = jit::to_string('r', inputs[i].get());
829 source_buffer << " const ";
830 jit::add_type<float> (source_buffer);
831 source_buffer << " " << registers[inputs[i].get()] << " = ";
832 if (thread_shared.contains(inputs[i].get())) {
833 source_buffer << jit::to_string('t', inputs[i].get())
834 << "[t_index]";
835 } else {
836 source_buffer << jit::to_string('v', inputs[i].get())
837 << "[";
838 if (state.get()) {
839 source_buffer << "offset + ";
840 }
841 source_buffer << "index]";
842 }
843 inputs[i]->endline(source_buffer, usage);
844 }
845#else
846 if (thread_shared.contains(inputs[i].get())) {
847 registers[inputs[i].get()] = jit::to_string('t', inputs[i].get()) + "[t_index]";
848 } else {
849 registers[inputs[i].get()] = jit::to_string('v', inputs[i].get()) + "["
850 + (state.get() ? "offset + " : "")
851 + "index]";
852 }
853#endif
854 }
855 }
856 if (state.get()) {
857#ifdef USE_INPUT_CACHE
858 registers[state.get()] = jit::to_string('r', state.get());
859 source_buffer << " device mt_state &" << registers[state.get()]
860 << " = " << jit::to_string('s', state.get())
861 << "[index]";
862 state->endline(source_buffer, usage);
863#else
864 registers[state.get()] = jit::to_string('s', state.get())
865 + "[index]";
866#endif
867 }
868 }
869
870//------------------------------------------------------------------------------
882//------------------------------------------------------------------------------
883 void create_kernel_postfix(std::ostringstream &source_buffer,
887 jit::register_map &registers,
888 const jit::register_usage &usage,
889 const jit::argument_set &thread_shared,
890 jit::register_map &thread_mem,
891 const size_t iterations=1) {
892 jit::argument_set out_registers;
893 for (auto &[out, in] : setters) {
894 if (!out->is_match(in)) {
895 auto a = out->compile(source_buffer, registers,
896 thread_mem, usage);
897 source_buffer << " ";
898 if (thread_shared.contains(in.get())) {
899 source_buffer << jit::to_string('t', in.get())
900 << "[t_index] = ";
901 } else {
902 source_buffer << jit::to_string('v', in.get())
903 << "[";
904 if (state.get()) {
905 source_buffer << "offset + ";
906 }
907 source_buffer << "index] = ";
908 }
909 if constexpr (SAFE_MATH) {
910 source_buffer << "isnan(" << registers[a.get()]
911 << ") ? 0.0 : ";
912 }
913 source_buffer << registers[a.get()] << ";" << std::endl;
914 out_registers.insert(out.get());
915 }
916 }
917
918 for (auto &out : outputs) {
919 if (!graph::variable_cast(out).get() &&
921 !out_registers.contains(out.get())) {
922 auto a = out->compile(source_buffer, registers,
923 thread_mem, usage);
924 source_buffer << " " << jit::to_string('o', out.get())
925 << "[";
926 if (state.get()) {
927 source_buffer << "offset + ";
928 }
929 source_buffer << "index] = ";
930 if constexpr (SAFE_MATH) {
931 if (!graph::random_cast(a).get()) {
932 source_buffer << "isnan(" << registers[a.get()]
933 << ") ? 0.0 : ";
934 }
935 }
936 source_buffer << registers[a.get()] << ";" << std::endl;
937 out_registers.insert(out.get());
938 }
939 }
940
941 if (iterations > 1) {
942 source_buffer << " }" << std::endl;
943 }
944 for (auto &[out, in] : setters) {
945 if (thread_shared.contains(in.get())) {
946 source_buffer << " "
947 << jit::to_string('v', in.get())
948 << "[";
949 if (state.get()) {
950 source_buffer << "offset + ";
951 }
952 source_buffer << "index] = "
953 << jit::to_string('t', in.get())
954 << "[t_index];" << std::endl;
955 }
956 }
957 source_buffer << " }" << std::endl << "}" << std::endl;
958 }
959
960//------------------------------------------------------------------------------
965//------------------------------------------------------------------------------
966 void create_reduction(std::ostringstream &source_buffer,
967 const size_t size) {
968 source_buffer << std::endl;
969 source_buffer << "kernel void max_reduction(" << std::endl;
970 source_buffer << " constant float *input [[buffer(0)]]," << std::endl;
971 source_buffer << " device float *result [[buffer(1)]]," << std::endl;
972 source_buffer << " uint i [[thread_position_in_grid]]," << std::endl;
973 source_buffer << " uint j [[simdgroup_index_in_threadgroup]]," << std::endl;
974 source_buffer << " uint k [[thread_index_in_simdgroup]]) {" << std::endl;
975 source_buffer << " if (i < " << size << ") {" << std::endl;
976 source_buffer << " float sub_max = input[i];" << std::endl;
977 source_buffer << " for (size_t index = i + 1024; index < " << size <<"; index += 1024) {" << std::endl;
978 source_buffer << " sub_max = max(sub_max, input[index]);" << std::endl;
979 source_buffer << " }" << std::endl;
980 source_buffer << " threadgroup float thread_max[32];" << std::endl;
981 source_buffer << " thread_max[j] = simd_max(sub_max);" << std::endl;
982 source_buffer << " threadgroup_barrier(mem_flags::mem_none);" << std::endl;
983 source_buffer << " if (j == 0) {" << std::endl;
984 source_buffer << " *result = simd_max(thread_max[k]);" << std::endl;
985 source_buffer << " }" << std::endl;
986 source_buffer << " }" << std::endl;
987 source_buffer << "}" << std::endl << std::endl;
988 }
989
990//------------------------------------------------------------------------------
994//------------------------------------------------------------------------------
996 return static_cast<float *> ([kernel_arguments[node.get()] contents]);
997 }
998 };
999}
1000
1001#endif /* metal_context_h */
Class representing a generic buffer.
Definition backend.hpp:29
std::function< void(void)> create_copy_call(graph::copy_nodes< float, SAFE_MATH > &setters)
Create kernel call that will to copy one buffer to another.
Definition metal_context.hpp:447
void create_reduction(std::ostringstream &source_buffer, const size_t size)
Create reduction.
Definition metal_context.hpp:966
void create_kernel_postfix(std::ostringstream &source_buffer, graph::output_nodes< float, SAFE_MATH > &outputs, graph::map_nodes< float, SAFE_MATH > &setters, graph::shared_random_state< float, SAFE_MATH > state, jit::register_map &registers, const jit::register_usage &usage, const jit::argument_set &thread_shared, jit::register_map &thread_mem, const size_t iterations=1)
Create kernel postfix.
Definition metal_context.hpp:883
static size_t max_concurrency()
Get the maximum number of concurrent instances.
Definition metal_context.hpp:56
MTLCompileOptions * compile_options()
Get the compile options.
Definition metal_context.hpp:491
void wait()
Hold the current thread until the command buffer has completed.
Definition metal_context.hpp:503
std::function< void(void)> run_function(std::function< void(void)> callback)
Run a callback function in the queue.
Definition metal_context.hpp:516
float check_value(const size_t index, const graph::shared_leaf< float, SAFE_MATH > &node)
Check the value.
Definition metal_context.hpp:556
void create_kernel_prefix(std::ostringstream &source_buffer, const std::string name, graph::input_nodes< float, SAFE_MATH > &inputs, graph::output_nodes< float, SAFE_MATH > &outputs, graph::atomic_nodes< float, SAFE_MATH > atomics, graph::shared_random_state< float, SAFE_MATH > &state, const size_t size, const std::vector< bool > &is_constant, jit::register_map &registers, const jit::register_usage &usage, jit::texture1d_list &textures1d, jit::texture2d_list &textures2d, jit::argument_set &thread_shared, jit::register_map &thread_mem, const size_t iterations=1)
Create kernel prefix.
Definition metal_context.hpp:623
void compile(const std::string kernel_source, std::vector< std::string > names, const bool add_reduction=false)
Compile the kernels.
Definition metal_context.hpp:83
void print_results(const size_t index, const graph::output_nodes< float, SAFE_MATH > &nodes)
Print out the results.
Definition metal_context.hpp:540
std::function< void(void)> create_kernel_call(const std::string kernel_name, graph::input_nodes< float, SAFE_MATH > inputs, graph::output_nodes< float, SAFE_MATH > outputs, graph::atomic_nodes< float, SAFE_MATH > atomics, graph::shared_random_state< float, SAFE_MATH > state, const size_t num_rays, const jit::texture1d_list &tex1d_list, const jit::texture2d_list &tex2d_list)
Create a kernel calling function.
Definition metal_context.hpp:120
float * get_buffer(graph::shared_leaf< float, SAFE_MATH > &node)
Get the buffer for a node.
Definition metal_context.hpp:995
static std::string device_type()
Device description.
Definition metal_context.hpp:63
void copy_to_device(graph::shared_leaf< float, SAFE_MATH > node, float *source)
Copy buffer contents to the device.
Definition metal_context.hpp:568
metal_context(const size_t index)
Construct a metal context.
Definition metal_context.hpp:72
static constexpr size_t random_state_size
Size of random state needed.
Definition metal_context.hpp:46
int remaining_const_memory
Remaining constant memory in bytes. NOT USED.
Definition metal_context.hpp:49
static constexpr size_t random_state_scale
Random state size multiplier.
Definition metal_context.hpp:44
std::function< float(void)> create_max_call(graph::shared_leaf< float, SAFE_MATH > &argument, std::function< void(void)> run)
Create a max compute kernel calling function.
Definition metal_context.hpp:340
void copy_to_host(graph::shared_leaf< float, SAFE_MATH > node, float *destination)
Copy buffer contents to host.
Definition metal_context.hpp:581
std::function< void(void)> create_zero_call(graph::input_nodes< float, SAFE_MATH > &inputs)
Create kernel call that will be memset a buffer to zero.
Definition metal_context.hpp:406
void create_header(std::ostringstream &source_buffer)
Create the source header.
Definition metal_context.hpp:598
A timing object.
Definition timing.hpp:18
void print() const
Print the result.
Definition timing.hpp:52
Name space for GPU backends.
Definition cpu_context.hpp:52
std::vector< shared_leaf< T, SAFE_MATH > > output_nodes
Convenience type alias for a vector of output nodes.
Definition node.hpp:705
std::vector< std::pair< shared_leaf< T, SAFE_MATH >, shared_variable< T, SAFE_MATH > > > map_nodes
Convenience type alias for mapping end codes back to inputs.
Definition node.hpp:2151
std::vector< std::pair< shared_variable< T, SAFE_MATH >, shared_variable< T, SAFE_MATH > > > copy_nodes
Convenience type alias for copying buffers.
Definition node.hpp:2155
shared_atomic_accumulate_1D< T, SAFE_MATH > atomic_accumulate_1D_cast(shared_leaf< T, SAFE_MATH > x)
Cast to a atomic accumulate 1D node.
Definition piecewise.hpp:2426
std::shared_ptr< leaf_node< T, SAFE_MATH > > shared_leaf
Convenience type alias for shared leaf nodes.
Definition node.hpp:690
input_nodes< T, SAFE_MATH > atomic_nodes
Convenience type alias for atomic inputd.
Definition node.hpp:2159
shared_variable< T, SAFE_MATH > variable_cast(shared_leaf< T, SAFE_MATH > x)
Cast to a variable node.
Definition node.hpp:2171
std::shared_ptr< random_state_node< T, SAFE_MATH > > shared_random_state
Convenience type alias for shared sqrt nodes.
Definition random.hpp:269
std::shared_ptr< variable_node< T, SAFE_MATH > > shared_variable
Convenience type alias for shared variable nodes.
Definition node.hpp:2145
shared_random< T, SAFE_MATH > random_cast(shared_leaf< T, SAFE_MATH > x)
Cast to a random node.
Definition random.hpp:539
std::vector< shared_variable< T, SAFE_MATH > > input_nodes
Convenience type alias for a vector of inputs.
Definition node.hpp:2148
std::unordered_map< void *, size_t > register_usage
Type alias for counting register usage.
Definition register.hpp:261
void add_type(std::basic_ostream< char > &stream)
Write out the node base type to a general stream.
Definition register.hpp:165
std::string smallest_uint_type(const size_t max_size)
Get smallest integer type.
Definition register.hpp:109
std::unordered_map< void *, std::string > register_map
Type alias for mapping node pointers to register names.
Definition register.hpp:259
std::unordered_map< void *, std::array< size_t, 2 > > texture2d_list
Type alias for indexing 2D textures.
Definition register.hpp:267
std::unordered_map< void *, size_t > texture1d_list
Type alias for indexing 1D textures.
Definition register.hpp:265
std::string to_string(const char prefix, const NODE *pointer)
Convert a graph::leaf_node pointer to a string.
Definition register.hpp:247
std::unordered_set< void * > argument_set
Type for tacking thread shared memory.
Definition register.hpp:271
Name space for output files.
Definition output.hpp:16
Piecewise nodes.
Random constants and distributions.
Routines to time the ray execution.