tvm
builtin.h
Go to the documentation of this file.
1 /*
2  * Licensed to the Apache Software Foundation (ASF) under one
3  * or more contributor license agreements. See the NOTICE file
4  * distributed with this work for additional information
5  * regarding copyright ownership. The ASF licenses this file
6  * to you under the Apache License, Version 2.0 (the
7  * "License"); you may not use this file except in compliance
8  * with the License. You may obtain a copy of the License at
9  *
10  * http://www.apache.org/licenses/LICENSE-2.0
11  *
12  * Unless required by applicable law or agreed to in writing,
13  * software distributed under the License is distributed on an
14  * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15  * KIND, either express or implied. See the License for the
16  * specific language governing permissions and limitations
17  * under the License.
18  */
19 
33 #ifndef TVM_TIR_BUILTIN_H_
34 #define TVM_TIR_BUILTIN_H_
35 
36 #include <tvm/ir/op.h>
37 #include <tvm/tir/expr.h>
38 
39 namespace tvm {
40 namespace tir {
41 
43 namespace builtin {
47 TVM_DLL const Op& ret();
51 TVM_DLL const Op& thread_return();
55 TVM_DLL const Op& reinterpret();
56 
60 TVM_DLL const Op& likely();
61 
65 TVM_DLL const Op& bitwise_and();
66 
70 TVM_DLL const Op& bitwise_or();
71 
75 TVM_DLL const Op& bitwise_xor();
76 
80 TVM_DLL const Op& bitwise_not();
81 
85 TVM_DLL const Op& shift_left();
86 
90 TVM_DLL const Op& shift_right();
91 
101 TVM_DLL const Op& large_uint_imm();
102 
109 TVM_DLL const Op& q_multiply_shift();
110 
123 TVM_DLL const Op& address_of();
124 
132 TVM_DLL const Op& if_then_else();
133 
141 TVM_DLL const Op& isnullptr();
142 
146 TVM_DLL const Op& isnan();
147 
151 TVM_DLL const Op& popcount();
152 
160 TVM_DLL const Op& fma();
161 
174 TVM_DLL const Op& call_extern();
175 
188 TVM_DLL const Op& call_pure_extern();
189 
200 TVM_DLL const Op& call_llvm_intrin();
201 
212 TVM_DLL const Op& call_llvm_pure_intrin();
213 
223 TVM_DLL const Op& call_spirv_pure_glsl450();
224 
225 // TODO(tvm-team) revisit the builtins below
226 // some of them can simply become ops with special codegen attr.
230 TVM_DLL const Op& prefetch();
231 
247 TVM_DLL const Op& tvm_access_ptr();
248 
253 TVM_DLL const Op& tvm_static_handle();
254 
259 TVM_DLL const Op& tvm_context_id();
260 
268 TVM_DLL const Op& tvm_tuple();
269 
277 TVM_DLL const Op& handle_add_byte_offset();
278 
287 TVM_DLL const Op& tvm_struct_get();
288 
297 TVM_DLL const Op& tvm_struct_set();
298 
305 TVM_DLL const Op& lookup_param();
306 
314 TVM_DLL const Op& tvm_throw_last_error();
315 
325 TVM_DLL const Op& tvm_stack_alloca();
326 
337 TVM_DLL const Op& tvm_stack_make_shape();
338 
358 TVM_DLL const Op& tvm_stack_make_array();
359 
372 TVM_DLL const Op& tvm_call_packed();
373 
383 TVM_DLL const Op& tvm_call_cpacked();
384 
396 TVM_DLL const Op& tvm_call_trace_packed();
397 
409 TVM_DLL const Op& tvm_thread_context();
410 
415 TVM_DLL const Op& tvm_thread_invariant();
416 
433 TVM_DLL const Op& tvm_call_packed_lowered();
434 
448 TVM_DLL const Op& tvm_call_cpacked_lowered();
449 
468 
477 TVM_DLL const Op& tvm_storage_sync();
478 
510 TVM_DLL const Op& tvm_warp_shuffle();
511 TVM_DLL const Op& tvm_warp_shuffle_up();
512 TVM_DLL const Op& tvm_warp_shuffle_down();
513 TVM_DLL const Op& tvm_warp_activemask();
514 
519 TVM_DLL const Op& tvm_global_barrier_kinit();
520 
532 TVM_DLL const Op& tvm_thread_allreduce();
533 
534 // TODO(tvm-team) TensorCore specific intrinsics should be directly registered under
535 // cuda. namespace and used through op.
548 TVM_DLL const Op& tvm_load_matrix_sync();
549 
561 TVM_DLL const Op& tvm_mma_sync();
562 
574 TVM_DLL const Op& tvm_bmma_sync();
575 
586 TVM_DLL const Op& tvm_fill_fragment();
587 
599 TVM_DLL const Op& tvm_store_matrix_sync();
600 
610 TVM_DLL const Op& ptx_mma();
611 
616 TVM_DLL const Op& ptx_ldg32();
617 
622 TVM_DLL const Op& ptx_ldg32();
623 
635 TVM_DLL const Op& ptx_mma_sp();
636 
644 TVM_DLL const Op& ptx_ldmatrix();
645 
655 TVM_DLL const Op& ptx_cp_async();
656 
667 TVM_DLL const Op& ptx_cp_async_bulk();
668 
676 TVM_DLL const Op& ptx_commit_group();
677 TVM_DLL const Op& ptx_wait_group();
678 
685 TVM_DLL const Op& ptx_cp_async_barrier();
686 
694 
701 TVM_DLL const Op& ptx_arrive_barrier();
702 
710 
717 TVM_DLL const Op& ptx_wait_barrier();
718 
725 TVM_DLL const Op& create_barriers();
726 
739 TVM_DLL const Op& mma_store();
740 
752 TVM_DLL const Op& mma_fill();
753 
754 // Metal SimdGroup matrix intrinsics
755 
765 
774 TVM_DLL const Op& simdgroup_load();
775 
784 TVM_DLL const Op& simdgroup_store();
785 
795 
796 // TODO(tvm-team) replace the usage of the vector operations by Shuffle.
800 TVM_DLL const Op& vectorhigh();
801 
805 TVM_DLL const Op& vectorlow();
806 
810 TVM_DLL const Op& vectorcombine();
811 
815 TVM_DLL const Op& dp4a();
816 
820 TVM_DLL const Op& atomic_add();
824 TVM_DLL const Op& nd_mem_alloc_with_scope();
825 
829 TVM_DLL const Op& texture2d_store();
830 
834 TVM_DLL const Op& texture2d_load();
835 
847 TVM_DLL const Op& dma_copy();
848 
855 TVM_DLL const Op& dma_wait();
856 
866 TVM_DLL const Op& dma_start_group();
867 
879 TVM_DLL const Op& dma_end_group();
880 
888 TVM_DLL const Op& assume();
889 
896 TVM_DLL const Op& undef();
897 
901 TVM_DLL const Op& start_profile_intrinsic();
902 
906 TVM_DLL const Op& end_profile_intrinsic();
907 
919 TVM_DLL const Op& anylist_getitem();
920 
932 TVM_DLL const Op& anylist_resetitem();
933 
946 
951 
956 TVM_DLL const Op& vscale();
957 
964 TVM_DLL const Op& get_active_lane_mask();
965 
967 TVM_DLL const Op& ignore_loop_partition();
968 
970 enum TVMStructFieldKind : int {
971  // array head address
984  // TVMValue field
990 };
991 } // namespace builtin
992 } // namespace tir
993 } // namespace tvm
994 #endif // TVM_TIR_BUILTIN_H_
Managed reference class to OpNode.
Definition: op.h:132
Primitive operators(builtin intrinsics) and registry for them.
const Op & tvm_call_packed_lowered()
Lowered version of call packed, the space of value and type codes are explicitly allocated.
const Op & tvm_thread_invariant()
Mark a condition to be thread invariant. This means the condition must be the same for all threads.
const Op & bitwise_not()
Bitwise not operator.
const Op & assume()
Provide a true statement that can be used for simplifications.
const Op & q_multiply_shift()
Execute a multiplication between two Q-numbers x and y followed by a right shift s The default roundi...
const Op & tvm_mma_sync()
tvm intrinsic for tensor core mma_sync operators.
const Op & bitwise_xor()
Bitwise xor operator.
const Op & bitwise_and()
Bitwise and operator.
const Op & dma_wait()
Wait until the number of DMA groups in flight is less than or equal to some maximum.
const Op & tvm_stack_alloca()
See pesudo code.
const Op & handle_add_byte_offset()
See pesudo code.
const Op & thread_return()
Return from a GPU thread.
const Op & simdgroup_store()
tvm intrinsic for storing data from simdgroup to device memory or threadgroup memory.
const Op & fma()
Fused multiply add.
const Op & tvm_call_cpacked()
See pesudo code.
const Op & ptx_commit_group()
tvm intrinsics for ptx async copy commit and wait.
const Op & popcount()
Popcount.
const Op & tvm_context_id()
Return a unique context id, used for hint of workspace separation. Different context id ganrantees no...
const Op & likely()
Marks a condition is likely going to happen.
const Op & tvm_struct_get()
See pesudo code.
const Op & shift_left()
Left shift.
const Op & tvm_stack_make_array()
Allocate a NDArray(DLTensor) on stack, return the handle.
const Op & ptx_cp_async_barrier()
tvm intrinsics for ptx async copy barrier using cp.async.mbarrier.arrive
const Op & tvm_call_packed()
See pesudo code.
const Op & create_barriers()
tvm intrinsics to create N barriers
const Op & mma_fill()
tvm intrinsic for zero-initializing an MMA accumulation register. For example, if each thread in a wa...
const Op & vectorcombine()
Concat two vectors.
const Op & tvm_call_cpacked_lowered()
Lowered version of call c-packed, the space of value and type codes are explicitly allocated.
const Op & large_uint_imm()
See pesudo code.
const Op & ptx_mma_sp()
tvm intrinsic for sparse tensor core ptx instructions.
const Op & dma_end_group()
End a group of DMA copies.
const Op & anylist_setitem_call_packed()
Set an item into any list by running packed function call.
const Op & tvm_access_ptr()
Get head access address with memory access pattern info.
const Op & vectorhigh()
Get the high level half of the vector.
const Op & prefetch()
same signature as llvm.prefetch
const Op & ptx_cp_async_bulk()
tvm intrinsics for ptx async copy from global to shared memory using cp.async.bulk
const Op & bitwise_or()
Bitwise or operator.
const Op & dp4a()
Dot product of two int8x4 vectors and add an optional accumulator.
const Op & tvm_fill_fragment()
tvm intrinsic for tensor core fill_fragment operators.
const Op & simdgroup_multiply_accumulate()
tvm intrinsic for multiply and accumulate two matrices in simdgroup
const Op & call_extern()
Call an extern C function with given name and signature from the types of args in the runtime environ...
const Op & tvm_static_handle()
Create a function local static handle that iniitalizes to nullptr. can be used to cache function loca...
const Op & tvm_thread_context()
See pesudo code Mark the content as thread local context, can get optimized by only call the call onc...
const Op & end_profile_intrinsic()
Profiling intrinsic.
const Op & tvm_struct_set()
See pesudo code.
const Op & texture2d_store()
Store to texture 2d memory.
const Op & isnan()
Check if value is nan.
const Op & address_of()
Returns the address of an element in the buffer (see pseudocode below).
const Op & simdgroup_load()
tvm intrinsic for loading data from device memory or threadgroup memory to simdgroup.
const Op & mma_store()
tvm intrinsic for storing the result of PTX MMA into a destination pointer. For example,...
const Op & undef()
Returns an initialized but arbitrary value.
const Op & ptx_ldg32()
tvm intrinsic for ptx predicate load with 32-bit data type.
const Op & reinterpret()
Reinterpret the value using the target type.
const Op & ptx_cp_async()
tvm intrinsics for ptx async copy from global to shared memory using cp.async
const Op & vectorlow()
Get the low-level half of the vector.
const Op & call_llvm_intrin()
Call an LLVM intrinsic with a given intrinsic id and signature from the types of args in the runtime ...
const Op & ptx_wait_barrier()
tvm intrinsics for ptx barrier wait using mbarrier.try_wait
const Op & tvm_bmma_sync()
tvm intrinsic for tensor core bmma_sync operators.
const Op & call_llvm_pure_intrin()
Call an LLVM pure intrinsic with a given intrinsic id and signature from the types of args in the run...
const Op & anylist_setitem_call_cpacked()
Same as anylist_setitem_call_packed but use C calling convention.
const Op & tvm_storage_sync()
See pseudo code.
const Op & tvm_throw_last_error()
See pesudo code.
const Op & tvm_load_matrix_sync()
tvm intrinsic for tensor core load operators.
const Op & nd_mem_alloc_with_scope()
Create an Nd memory allocation with storage scope.
const Op & tvm_thread_allreduce()
See pesudo code.
const Op & isnullptr()
See pesudo code.
const Op & start_profile_intrinsic()
Profiling intrinsic.
const Op & tvm_call_trace_packed_lowered()
Lowered version of trace intrinsic, the space of value and type codes are explicitly allocated....
const Op & anylist_resetitem()
Reset and clear a item in any list.
const Op & vscale()
Get the target's vscale value. It will be lowered to llvm.vscale intrinsic (https://llvm....
const Op & ptx_arrive_barrier_expect_tx()
tvm intrinsic for ptx barrier arrival with expect tx using mbarrier.arrive.expect_tx
const Op & tvm_tuple()
tvm_tuple is not an actual function and cannot codegen. It is used to represent tuple structure in va...
const Op & atomic_add()
atomic add instruction, corresponding e.g. to atomicAdd in CUDA
const Op & anylist_getitem()
Get a item from any list and return it.
const Op & tvm_stack_make_shape()
Allocate a shape tuple on stack, return the handle.
const Op & ptx_arrive_barrier()
tvm intrinsics for ptx barrier arrival using mbarrier.arrive
const Op & call_spirv_pure_glsl450()
Call an SPIRV pure GLSL450 intrinsic.
const Op & tvm_call_trace_packed()
See pesudo code.
const Op & tvm_global_barrier_kinit()
Initialize the global barrier. Call this at beginning of kernel that need global barrier.
const Op & tvm_warp_shuffle()
See pseudo code.
const Op & ptx_init_barrier_thread_count()
tvm intrinsics for ptx barrier initialization of thread count using mbarrier.init
const Op & ignore_loop_partition()
Annotate a predicate not be considered as target condition of loop partition.
const Op & dma_start_group()
Start a group of DMA copies.
const Op & get_active_lane_mask()
Calculate a predicate mask given an upper bound (limit) and a current value (base).
TVMStructFieldKind
The kind of structure field info used in intrinsic.
Definition: builtin.h:970
@ kArrAddr
Definition: builtin.h:972
@ kArrTypeLanes
Definition: builtin.h:979
@ kArrTypeBits
Definition: builtin.h:978
@ kArrKindBound_
Definition: builtin.h:983
@ kArrShape
Definition: builtin.h:974
@ kArrTypeCode
Definition: builtin.h:977
@ kTVMValueContent
Definition: builtin.h:985
@ kTVMFFIAnyTypeIndex
Definition: builtin.h:986
@ kArrData
Definition: builtin.h:973
@ kArrDeviceId
Definition: builtin.h:981
@ kTVMFFIAnyUnionValue
Definition: builtin.h:988
@ kArrStrides
Definition: builtin.h:975
@ kTVMValueKindBound_
Definition: builtin.h:989
@ kTVMFFIAnyZeroPadding
Definition: builtin.h:987
@ kArrDeviceType
Definition: builtin.h:982
@ kArrNDim
Definition: builtin.h:976
@ kArrByteOffset
Definition: builtin.h:980
const Op & ptx_mma()
tvm intrinsic for ptx tensor core mma instructions.
const Op & tvm_warp_shuffle_up()
const Op & call_pure_extern()
Call an pure extern C function with given name and signature from the types of args in the runtime en...
const Op & texture2d_load()
Load from texture 2d memory.
const Op & if_then_else()
Same as select, used for unsafe memory access.
const Op & ret()
Return value.
const Op & ptx_ldmatrix()
tvm intrinsic for ptx load matrix from shared memory.
const Op & ptx_wait_group()
const Op & shift_right()
Right shift.
const Op & lookup_param()
See pseudo code Type lookup_param(String param_name) { return __tvm_param__param_name; }.
const Op & tvm_warp_shuffle_down()
const Op & make_filled_simdgroup_matrix()
tvm intrinsic for initializing and simdgroup with given value.
const Op & tvm_warp_activemask()
const Op & tvm_store_matrix_sync()
tvm intrinsic for tensor core store operators.
const Op & dma_copy()
Initiate a non-blocking DMA copy from source to destination.
Performance counters for profiling via the PAPI library.
Definition: analyzer.h:37
TIR expressions.