diff options
Diffstat (limited to 'mlir')
88 files changed, 2540 insertions, 739 deletions
diff --git a/mlir/docs/PassManagement.md b/mlir/docs/PassManagement.md index c9d705f..e9ecb99 100644 --- a/mlir/docs/PassManagement.md +++ b/mlir/docs/PassManagement.md @@ -1124,17 +1124,44 @@ pipeline. This display mode is available in mlir-opt via $ mlir-opt foo.mlir -mlir-disable-threading -pass-pipeline='builtin.module(func.func(cse,canonicalize),convert-func-to-llvm)' -mlir-timing -mlir-timing-display=list ===-------------------------------------------------------------------------=== - ... Pass execution timing report ... + ... Execution time report ... ===-------------------------------------------------------------------------=== - Total Execution Time: 0.0203 seconds - - ---Wall Time--- --- Name --- - 0.0047 ( 55.9%) Canonicalizer - 0.0019 ( 22.2%) VerifierPass - 0.0016 ( 18.5%) LLVMLoweringPass - 0.0003 ( 3.4%) CSE - 0.0002 ( 1.9%) (A) DominanceInfo - 0.0084 (100.0%) Total + Total Execution Time: 0.0135 seconds + + ----Wall Time---- ----Name---- + 0.0135 (100.0%) root + 0.0041 ( 30.1%) Parser + 0.0018 ( 13.3%) ConvertFuncToLLVMPass + 0.0011 ( 8.2%) Output + 0.0007 ( 5.2%) Pipeline Collection : ['func.func'] + 0.0006 ( 4.6%) 'func.func' Pipeline + 0.0005 ( 3.5%) Canonicalizer + 0.0001 ( 0.9%) CSE + 0.0001 ( 0.5%) (A) DataLayoutAnalysis + 0.0000 ( 0.1%) (A) DominanceInfo + 0.0058 ( 43.2%) Rest + 0.0135 (100.0%) Total +``` + +The results can be displayed in JSON format via `-mlir-output-format=json`. + +```shell +$ mlir-opt foo.mlir -mlir-disable-threading -pass-pipeline='builtin.module(func.func(cse,canonicalize),convert-func-to-llvm)' -mlir-timing -mlir-timing-display=list -mlir-output-format=json + +[ +{"wall": {"duration": 0.0135, "percentage": 100.0}, "name": "root"}, +{"wall": {"duration": 0.0041, "percentage": 30.1}, "name": "Parser"}, +{"wall": {"duration": 0.0018, "percentage": 13.3}, "name": "ConvertFuncToLLVMPass"}, +{"wall": {"duration": 0.0011, "percentage": 8.2}, "name": "Output"}, +{"wall": {"duration": 0.0007, "percentage": 5.2}, "name": "Pipeline Collection : ['func.func']"}, +{"wall": {"duration": 0.0006, "percentage": 4.6}, "name": "'func.func' Pipeline"}, +{"wall": {"duration": 0.0005, "percentage": 3.5}, "name": "Canonicalizer"}, +{"wall": {"duration": 0.0001, "percentage": 0.9}, "name": "CSE"}, +{"wall": {"duration": 0.0001, "percentage": 0.5}, "name": "(A) DataLayoutAnalysis"}, +{"wall": {"duration": 0.0000, "percentage": 0.1}, "name": "(A) DominanceInfo"}, +{"wall": {"duration": 0.0058, "percentage": 43.2}, "name": "Rest"}, +{"wall": {"duration": 0.0135, "percentage": 100.0}, "name": "Total"} +] ``` ##### Tree Display Mode @@ -1149,21 +1176,48 @@ invalidated and recomputed. This is the default display mode. $ mlir-opt foo.mlir -mlir-disable-threading -pass-pipeline='builtin.module(func.func(cse,canonicalize),convert-func-to-llvm)' -mlir-timing ===-------------------------------------------------------------------------=== - ... Pass execution timing report ... + ... Execution time report ... ===-------------------------------------------------------------------------=== - Total Execution Time: 0.0249 seconds - - ---Wall Time--- --- Name --- - 0.0058 ( 70.8%) 'func.func' Pipeline - 0.0004 ( 4.3%) CSE - 0.0002 ( 2.6%) (A) DominanceInfo - 0.0004 ( 4.8%) VerifierPass - 0.0046 ( 55.4%) Canonicalizer - 0.0005 ( 6.2%) VerifierPass - 0.0005 ( 5.8%) VerifierPass - 0.0014 ( 17.2%) LLVMLoweringPass - 0.0005 ( 6.2%) VerifierPass - 0.0082 (100.0%) Total + Total Execution Time: 0.0127 seconds + + ----Wall Time---- ----Name---- + 0.0038 ( 30.2%) Parser + 0.0006 ( 4.8%) 'func.func' Pipeline + 0.0001 ( 0.9%) CSE + 0.0000 ( 0.1%) (A) DominanceInfo + 0.0005 ( 3.7%) Canonicalizer + 0.0017 ( 13.7%) ConvertFuncToLLVMPass + 0.0001 ( 0.6%) (A) DataLayoutAnalysis + 0.0010 ( 8.2%) Output + 0.0054 ( 42.5%) Rest + 0.0127 (100.0%) Total +``` + +The results can be displayed in JSON format via `-mlir-output-format=json`. + +```shell +$ mlir-opt foo.mlir -mlir-disable-threading -pass-pipeline='builtin.module(func.func(cse,canonicalize),convert-func-to-llvm)' -mlir-timing -mlir-output-format=json + +[ +{"wall": {"duration": 0.0038, "percentage": 30.2}, "name": "Parser", "passes": [ +{}]}, +{"wall": {"duration": 0.0006, "percentage": 4.8}, "name": "'func.func' Pipeline", "passes": [ + {"wall": {"duration": 0.0001, "percentage": 0.9}, "name": "CSE", "passes": [ + {"wall": {"duration": 0.0000, "percentage": 0.1}, "name": "(A) DominanceInfo", "passes": [ + {}]}, + {}]}, + {"wall": {"duration": 0.0005, "percentage": 3.7}, "name": "Canonicalizer", "passes": [ + {}]}, +{}]}, +{"wall": {"duration": 0.0017, "percentage": 13.7}, "name": "ConvertFuncToLLVMPass", "passes": [ + {"wall": {"duration": 0.0001, "percentage": 0.6}, "name": "(A) DataLayoutAnalysis", "passes": [ + {}]}, +{}]}, +{"wall": {"duration": 0.0010, "percentage": 8.2}, "name": "Output", "passes": [ +{}]}, +{"wall": {"duration": 0.0054, "percentage": 42.5}, "name": "Rest"}, +{"wall": {"duration": 0.0127, "percentage": 100.0}, "name": "Total"} +] ``` ##### Multi-threaded Pass Timing diff --git a/mlir/include/mlir/Dialect/Affine/LoopUtils.h b/mlir/include/mlir/Dialect/Affine/LoopUtils.h index 723a262..d143954 100644 --- a/mlir/include/mlir/Dialect/Affine/LoopUtils.h +++ b/mlir/include/mlir/Dialect/Affine/LoopUtils.h @@ -299,53 +299,8 @@ LogicalResult separateFullTiles(MutableArrayRef<AffineForOp> nest, SmallVectorImpl<AffineForOp> *fullTileNest = nullptr); -/// Walk either an scf.for or an affine.for to find a band to coalesce. -template <typename LoopOpTy> -LogicalResult coalescePerfectlyNestedLoops(LoopOpTy op) { - LogicalResult result(failure()); - SmallVector<LoopOpTy> loops; - getPerfectlyNestedLoops(loops, op); - - // Look for a band of loops that can be coalesced, i.e. perfectly nested - // loops with bounds defined above some loop. - // 1. For each loop, find above which parent loop its operands are - // defined. - SmallVector<unsigned, 4> operandsDefinedAbove(loops.size()); - for (unsigned i = 0, e = loops.size(); i < e; ++i) { - operandsDefinedAbove[i] = i; - for (unsigned j = 0; j < i; ++j) { - if (areValuesDefinedAbove(loops[i].getOperands(), loops[j].getRegion())) { - operandsDefinedAbove[i] = j; - break; - } - } - } - - // 2. Identify bands of loops such that the operands of all of them are - // defined above the first loop in the band. Traverse the nest bottom-up - // so that modifications don't invalidate the inner loops. - for (unsigned end = loops.size(); end > 0; --end) { - unsigned start = 0; - for (; start < end - 1; ++start) { - auto maxPos = - *std::max_element(std::next(operandsDefinedAbove.begin(), start), - std::next(operandsDefinedAbove.begin(), end)); - if (maxPos > start) - continue; - assert(maxPos == start && - "expected loop bounds to be known at the start of the band"); - auto band = llvm::MutableArrayRef(loops.data() + start, end - start); - if (succeeded(coalesceLoops(band))) - result = success(); - break; - } - // If a band was found and transformed, keep looking at the loops above - // the outermost transformed loop. - if (start != end - 1) - end = start + 1; - } - return result; -} +/// Walk an affine.for to find a band to coalesce. +LogicalResult coalescePerfectlyNestedAffineLoops(AffineForOp op); } // namespace affine } // namespace mlir diff --git a/mlir/include/mlir/Dialect/DLTI/DLTI.h b/mlir/include/mlir/Dialect/DLTI/DLTI.h index bf23aa2..5ac7c11 100644 --- a/mlir/include/mlir/Dialect/DLTI/DLTI.h +++ b/mlir/include/mlir/Dialect/DLTI/DLTI.h @@ -100,6 +100,9 @@ public: /// Returns the list of entries. DataLayoutEntryListRef getEntries() const; + /// Returns the endiannes identifier. + StringAttr getEndiannessIdentifier(MLIRContext *context) const; + /// Returns the alloca memory space identifier. StringAttr getAllocaMemorySpaceIdentifier(MLIRContext *context) const; diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h index 725a1bc..c039156 100644 --- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h +++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.h @@ -30,8 +30,14 @@ namespace mlir { namespace emitc { void buildTerminatedBody(OpBuilder &builder, Location loc); + /// Determines whether \p type is a valid integer type in EmitC. bool isSupportedIntegerType(mlir::Type type); + +/// Determines whether \p type is integer like, i.e. it's a supported integer, +/// an index or opaque type. +bool isIntegerIndexOrOpaqueType(Type type); + /// Determines whether \p type is a valid floating-point type in EmitC. bool isSupportedFloatType(mlir::Type type); } // namespace emitc diff --git a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td index d746222..e611fd2 100644 --- a/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td +++ b/mlir/include/mlir/Dialect/EmitC/IR/EmitC.td @@ -347,9 +347,8 @@ def EmitC_ConstantOp : EmitC_Op<"constant", [ConstantLike]> { %0 = "emitc.constant"(){value = 42 : i32} : () -> i32 // Constant emitted as `char = CHAR_MIN;` - %1 = "emitc.constant"() - {value = #emitc.opaque<"CHAR_MIN"> : !emitc.opaque<"char">} - : () -> !emitc.opaque<"char"> + %1 = "emitc.constant"() {value = #emitc.opaque<"CHAR_MIN">} + : () -> !emitc.opaque<"char"> ``` }]; @@ -992,9 +991,8 @@ def EmitC_VariableOp : EmitC_Op<"variable", []> { %0 = "emitc.variable"(){value = 42 : i32} : () -> i32 // Variable emitted as `int32_t* = NULL;` - %1 = "emitc.variable"() - {value = #emitc.opaque<"NULL"> : !emitc.opaque<"int32_t*">} - : () -> !emitc.opaque<"int32_t*"> + %1 = "emitc.variable"() {value = #emitc.opaque<"NULL">} + : () -> !emitc.ptr<!emitc.opaque<"int32_t">> ``` Since folding is not supported, it can be used with pointers. @@ -1155,35 +1153,41 @@ def EmitC_IfOp : EmitC_Op<"if", let hasCustomAssemblyFormat = 1; } -def EmitC_SubscriptOp : EmitC_Op<"subscript", - [TypesMatchWith<"result type matches element type of 'array'", - "array", "result", - "::llvm::cast<ArrayType>($_self).getElementType()">]> { - let summary = "Array subscript operation"; +def EmitC_SubscriptOp : EmitC_Op<"subscript", []> { + let summary = "Subscript operation"; let description = [{ With the `subscript` operation the subscript operator `[]` can be applied - to variables or arguments of array type. + to variables or arguments of array, pointer and opaque type. Example: ```mlir %i = index.constant 1 %j = index.constant 7 - %0 = emitc.subscript %arg0[%i, %j] : <4x8xf32>, index, index + %0 = emitc.subscript %arg0[%i, %j] : !emitc.array<4x8xf32>, index, index + %1 = emitc.subscript %arg1[%i] : !emitc.ptr<i32>, index ``` }]; - let arguments = (ins Arg<EmitC_ArrayType, "the reference to load from">:$array, - Variadic<IntegerIndexOrOpaqueType>:$indices); + let arguments = (ins Arg<AnyTypeOf<[ + EmitC_ArrayType, + EmitC_OpaqueType, + EmitC_PointerType]>, + "the value to subscript">:$value, + Variadic<AnyType>:$indices); let results = (outs AnyType:$result); let builders = [ - OpBuilder<(ins "Value":$array, "ValueRange":$indices), [{ - build($_builder, $_state, cast<ArrayType>(array.getType()).getElementType(), array, indices); + OpBuilder<(ins "TypedValue<ArrayType>":$array, "ValueRange":$indices), [{ + build($_builder, $_state, array.getType().getElementType(), array, indices); + }]>, + OpBuilder<(ins "TypedValue<PointerType>":$pointer, "Value":$index), [{ + build($_builder, $_state, pointer.getType().getPointee(), pointer, + ValueRange{index}); }]> ]; let hasVerifier = 1; - let assemblyFormat = "$array `[` $indices `]` attr-dict `:` type($array) `,` type($indices)"; + let assemblyFormat = "$value `[` $indices `]` attr-dict `:` functional-type(operands, results)"; } diff --git a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td index 28526f1..a52cca3 100644 --- a/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td +++ b/mlir/include/mlir/Dialect/LLVMIR/LLVMIntrinsicOps.td @@ -562,8 +562,10 @@ class LLVM_DbgIntrOp<string name, string argName, list<Trait> traits = []> }]; } -def LLVM_DbgDeclareOp : LLVM_DbgIntrOp<"dbg.declare", "addr", - [DeclareOpInterfaceMethods<PromotableOpInterface>]> { +def LLVM_DbgDeclareOp : LLVM_DbgIntrOp<"dbg.declare", "addr", [ + DeclareOpInterfaceMethods<PromotableOpInterface, [ + "requiresReplacedValues", "visitReplacedValues" + ]>]> { let summary = "Describes how the address relates to a source language variable."; let arguments = (ins LLVM_AnyPointer:$addr, diff --git a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td index f33942b..4574518 100644 --- a/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td +++ b/mlir/include/mlir/Dialect/OpenMP/OpenMPOps.td @@ -2135,8 +2135,8 @@ def DeclareReductionOp : OpenMP_Op<"declare_reduction", [Symbol, let summary = "declares a reduction kind"; let description = [{ - Declares an OpenMP reduction kind. This requires two mandatory and one - optional region. + Declares an OpenMP reduction kind. This requires two mandatory and two + optional regions. 1. The initializer region specifies how to initialize the thread-local reduction value. This is usually the neutral element of the reduction. @@ -2149,6 +2149,10 @@ def DeclareReductionOp : OpenMP_Op<"declare_reduction", [Symbol, 3. The atomic reduction region is optional and specifies how two values can be combined atomically given local accumulator variables. It is expected to store the combined value in the first accumulator variable. + 4. The cleanup region is optional and specifies how to clean up any memory + allocated by the initializer region. The region has an argument that + contains the value of the thread-local reduction accumulator. This will + be executed after the reduction has completed. Note that the MLIR type system does not allow for type-polymorphic reductions. Separate reduction declarations should be created for different @@ -2163,12 +2167,14 @@ def DeclareReductionOp : OpenMP_Op<"declare_reduction", [Symbol, let regions = (region AnyRegion:$initializerRegion, AnyRegion:$reductionRegion, - AnyRegion:$atomicReductionRegion); + AnyRegion:$atomicReductionRegion, + AnyRegion:$cleanupRegion); let assemblyFormat = "$sym_name `:` $type attr-dict-with-keyword " "`init` $initializerRegion " "`combiner` $reductionRegion " - "custom<AtomicReductionRegion>($atomicReductionRegion)"; + "custom<AtomicReductionRegion>($atomicReductionRegion) " + "custom<CleanupReductionRegion>($cleanupRegion)"; let extraClassDeclaration = [{ PointerLikeType getAccumulatorType() { diff --git a/mlir/include/mlir/Dialect/SCF/Utils/Utils.h b/mlir/include/mlir/Dialect/SCF/Utils/Utils.h index 883d11b..bc09cc7 100644 --- a/mlir/include/mlir/Dialect/SCF/Utils/Utils.h +++ b/mlir/include/mlir/Dialect/SCF/Utils/Utils.h @@ -100,11 +100,16 @@ getSCFMinMaxExpr(Value value, SmallVectorImpl<Value> &dims, /// `loops` contains a list of perfectly nested loops with bounds and steps /// independent of any loop induction variable involved in the nest. LogicalResult coalesceLoops(MutableArrayRef<scf::ForOp> loops); +LogicalResult coalesceLoops(RewriterBase &rewriter, + MutableArrayRef<scf::ForOp>); + +/// Walk an affine.for to find a band to coalesce. +LogicalResult coalescePerfectlyNestedSCFForLoops(scf::ForOp op); /// Take the ParallelLoop and for each set of dimension indices, combine them /// into a single dimension. combinedDimensions must contain each index into /// loops exactly once. -void collapseParallelLoops(scf::ParallelOp loops, +void collapseParallelLoops(RewriterBase &rewriter, scf::ParallelOp loops, ArrayRef<std::vector<unsigned>> combinedDimensions); /// Unrolls this for operation by the specified unroll factor. Returns failure diff --git a/mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h b/mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h index 31e19ff..67a6581 100644 --- a/mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h +++ b/mlir/include/mlir/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.h @@ -29,9 +29,12 @@ struct ValueBoundsConstraintSet : protected ::mlir::ValueBoundsConstraintSet { struct ScalableValueBoundsConstraintSet : public llvm::RTTIExtends<ScalableValueBoundsConstraintSet, detail::ValueBoundsConstraintSet> { - ScalableValueBoundsConstraintSet(MLIRContext *context, unsigned vscaleMin, - unsigned vscaleMax) - : RTTIExtends(context), vscaleMin(vscaleMin), vscaleMax(vscaleMax){}; + ScalableValueBoundsConstraintSet( + MLIRContext *context, + ValueBoundsConstraintSet::StopConditionFn stopCondition, + unsigned vscaleMin, unsigned vscaleMax) + : RTTIExtends(context, stopCondition), vscaleMin(vscaleMin), + vscaleMax(vscaleMax) {}; using RTTIExtends::bound; using RTTIExtends::StopConditionFn; diff --git a/mlir/include/mlir/IR/PatternMatch.h b/mlir/include/mlir/IR/PatternMatch.h index 15b1c38..2562301 100644 --- a/mlir/include/mlir/IR/PatternMatch.h +++ b/mlir/include/mlir/IR/PatternMatch.h @@ -15,6 +15,7 @@ #include "llvm/Support/TypeName.h" #include <optional> +using llvm::SmallPtrSetImpl; namespace mlir { class PatternRewriter; @@ -704,6 +705,8 @@ public: return user != exceptedUser; }); } + void replaceAllUsesExcept(Value from, Value to, + const SmallPtrSetImpl<Operation *> &preservedUsers); /// Used to notify the listener that the IR failed to be rewritten because of /// a match failure, and provide a callback to populate a diagnostic with the diff --git a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h index 0463546..76bf33e 100644 --- a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h +++ b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.h @@ -64,6 +64,10 @@ std::optional<uint64_t> getDefaultIndexBitwidth(Type type, const DataLayout &dataLayout, ArrayRef<DataLayoutEntryInterface> params); +/// Default handler for endianness request. Dispatches to the +/// DataLayoutInterface if specified, otherwise returns the default. +Attribute getDefaultEndianness(DataLayoutEntryInterface entry); + /// Default handler for alloca memory space request. Dispatches to the /// DataLayoutInterface if specified, otherwise returns the default. Attribute getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry); @@ -192,6 +196,9 @@ public: /// type is not a pointer-like type, it returns std::nullopt. std::optional<uint64_t> getTypeIndexBitwidth(Type t) const; + /// Returns the specified endianness. + Attribute getEndianness() const; + /// Returns the memory space used for AllocaOps. Attribute getAllocaMemorySpace() const; @@ -230,6 +237,8 @@ private: mutable DenseMap<Type, uint64_t> preferredAlignments; mutable DenseMap<Type, std::optional<uint64_t>> indexBitwidths; + /// Cache for the endianness. + mutable std::optional<Attribute> endianness; /// Cache for alloca, global, and program memory spaces. mutable std::optional<Attribute> allocaMemorySpace; mutable std::optional<Attribute> programMemorySpace; diff --git a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td index 0ee7a11..9edc885 100644 --- a/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td +++ b/mlir/include/mlir/Interfaces/DataLayoutInterfaces.td @@ -107,6 +107,12 @@ def DataLayoutSpecInterface : AttrInterface<"DataLayoutSpecInterface"> { /*args=*/(ins) >, InterfaceMethod< + /*description=*/"Returns the endianness identifier.", + /*retTy=*/"::mlir::StringAttr", + /*methodName=*/"getEndiannessIdentifier", + /*args=*/(ins "::mlir::MLIRContext *":$context) + >, + InterfaceMethod< /*description=*/"Returns the alloca memory space identifier.", /*retTy=*/"::mlir::StringAttr", /*methodName=*/"getAllocaMemorySpaceIdentifier", @@ -297,6 +303,18 @@ def DataLayoutOpInterface : OpInterface<"DataLayoutOpInterface"> { }] >, StaticInterfaceMethod< + /*description=*/"Returns the endianness used by the ABI computed " + "using the relevant entries. The data layout object " + "can be used for recursive queries.", + /*retTy=*/"::mlir::Attribute", + /*methodName=*/"getEndianness", + /*args=*/(ins "::mlir::DataLayoutEntryInterface":$entry), + /*methodBody=*/"", + /*defaultImplementation=*/[{ + return ::mlir::detail::getDefaultEndianness(entry); + }] + >, + StaticInterfaceMethod< /*description=*/"Returns the memory space used by the ABI computed " "using the relevant entries. The data layout object " "can be used for recursive queries.", diff --git a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td index e10e2d4..9db8936 100644 --- a/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td +++ b/mlir/include/mlir/Interfaces/MemorySlotInterfaces.td @@ -229,6 +229,36 @@ def PromotableOpInterface : OpInterface<"PromotableOpInterface"> { (ins "const ::llvm::SmallPtrSetImpl<mlir::OpOperand *> &":$blockingUses, "::mlir::RewriterBase &":$rewriter) >, + InterfaceMethod<[{ + This method allows the promoted operation to visit the SSA values used + in place of the memory slot once the promotion process of the memory + slot is complete. + + If this method returns true, the `visitReplacedValues` method on this + operation will be called after the main mutation stage finishes + (i.e., after all ops have been processed with `removeBlockingUses`). + + Operations should only the replaced values if the intended + transformation applies to all the replaced values. Furthermore, replaced + values must not be deleted. + }], "bool", "requiresReplacedValues", (ins), [{}], + [{ return false; }] + >, + InterfaceMethod<[{ + Transforms the IR using the SSA values that replaced the memory slot. + + This method will only be called after all blocking uses have been + scheduled for removal and if `requiresReplacedValues` returned + true. + + The rewriter is located after the promotable operation on call. All IR + mutations must happen through the rewriter. During the transformation, + *no operation should be deleted*. + }], + "void", "visitReplacedValues", + (ins "::llvm::ArrayRef<std::pair<::mlir::Operation*, ::mlir::Value>>":$mutatedDefs, + "::mlir::RewriterBase &":$rewriter), [{}], [{ return; }] + >, ]; } diff --git a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h index bdfd689..83107a3 100644 --- a/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h +++ b/mlir/include/mlir/Interfaces/ValueBoundsOpInterface.h @@ -117,8 +117,9 @@ public: /// /// The first parameter of the function is the shaped value/index-typed /// value. The second parameter is the dimension in case of a shaped value. - using StopConditionFn = - function_ref<bool(Value, std::optional<int64_t> /*dim*/)>; + /// The third parameter is this constraint set. + using StopConditionFn = std::function<bool( + Value, std::optional<int64_t> /*dim*/, ValueBoundsConstraintSet &cstr)>; /// Compute a bound for the given index-typed value or shape dimension size. /// The computed bound is stored in `resultMap`. The operands of the bound are @@ -271,22 +272,20 @@ protected: /// An index-typed value or the dimension of a shaped-type value. using ValueDim = std::pair<Value, int64_t>; - ValueBoundsConstraintSet(MLIRContext *ctx); + ValueBoundsConstraintSet(MLIRContext *ctx, StopConditionFn stopCondition); /// Populates the constraint set for a value/map without actually computing /// the bound. Returns the position for the value/map (via the return value /// and `posOut` output parameter). int64_t populateConstraintsSet(Value value, - std::optional<int64_t> dim = std::nullopt, - StopConditionFn stopCondition = nullptr); + std::optional<int64_t> dim = std::nullopt); int64_t populateConstraintsSet(AffineMap map, ValueDimList mapOperands, - StopConditionFn stopCondition = nullptr, int64_t *posOut = nullptr); /// Iteratively process all elements on the worklist until an index-typed /// value or shaped value meets `stopCondition`. Such values are not processed /// any further. - void processWorklist(StopConditionFn stopCondition); + void processWorklist(); /// Bound the given column in the underlying constraint set by the given /// expression. @@ -333,6 +332,9 @@ protected: /// Builder for constructing affine expressions. Builder builder; + + /// The current stop condition function. + StopConditionFn stopCondition = nullptr; }; } // namespace mlir diff --git a/mlir/include/mlir/Pass/Pass.h b/mlir/include/mlir/Pass/Pass.h index 0f50f30..e71c49a 100644 --- a/mlir/include/mlir/Pass/Pass.h +++ b/mlir/include/mlir/Pass/Pass.h @@ -355,7 +355,7 @@ private: template <typename OpT = void> class OperationPass : public Pass { public: - ~OperationPass() = default; + ~OperationPass() override = default; protected: OperationPass(TypeID passID) : Pass(passID, OpT::getOperationName()) {} @@ -400,7 +400,7 @@ protected: template <> class OperationPass<void> : public Pass { public: - ~OperationPass() = default; + ~OperationPass() override = default; protected: OperationPass(TypeID passID) : Pass(passID) {} @@ -461,7 +461,7 @@ public: static bool classof(const Pass *pass) { return pass->getTypeID() == TypeID::get<PassT>(); } - ~PassWrapper() = default; + ~PassWrapper() override = default; protected: PassWrapper() : BaseT(TypeID::get<PassT>()) {} diff --git a/mlir/include/mlir/Support/Timing.h b/mlir/include/mlir/Support/Timing.h index bc3a642..a8a4bfd 100644 --- a/mlir/include/mlir/Support/Timing.h +++ b/mlir/include/mlir/Support/Timing.h @@ -321,6 +321,53 @@ private: }; //===----------------------------------------------------------------------===// +// OutputStrategy +//===----------------------------------------------------------------------===// + +/// Simple record class to record timing information. +struct TimeRecord { + TimeRecord(double wall = 0.0, double user = 0.0) : wall(wall), user(user) {} + + TimeRecord &operator+=(const TimeRecord &other) { + wall += other.wall; + user += other.user; + return *this; + } + + TimeRecord &operator-=(const TimeRecord &other) { + wall -= other.wall; + user -= other.user; + return *this; + } + + double wall, user; +}; + +/// Facilities for printing timing reports to various output formats. +/// +/// This is an abstract class that serves as the foundation for printing. +/// Users can implement additional output formats by extending this abstract +/// class. +class OutputStrategy { +public: + OutputStrategy(raw_ostream &os) : os(os) {} + virtual ~OutputStrategy() = default; + + virtual void printHeader(const TimeRecord &total) = 0; + virtual void printFooter() = 0; + virtual void printTime(const TimeRecord &time, const TimeRecord &total) = 0; + virtual void printListEntry(StringRef name, const TimeRecord &time, + const TimeRecord &total, + bool lastEntry = false) = 0; + virtual void printTreeEntry(unsigned indent, StringRef name, + const TimeRecord &time, + const TimeRecord &total) = 0; + virtual void printTreeEntryEnd(unsigned indent, bool lastEntry = false) = 0; + + raw_ostream &os; +}; + +//===----------------------------------------------------------------------===// // DefaultTimingManager //===----------------------------------------------------------------------===// @@ -351,6 +398,15 @@ public: Tree, }; + /// The different output formats for printing the timers. + enum class OutputFormat { + /// In this format the results are displayed in text format. + Text, + + /// In this format the results are displayed in JSON format. + Json, + }; + DefaultTimingManager(); DefaultTimingManager(DefaultTimingManager &&rhs); ~DefaultTimingManager() override; @@ -372,10 +428,7 @@ public: DisplayMode getDisplayMode() const; /// Change the stream where the output will be printed to. - void setOutput(raw_ostream &os); - - /// Return the current output stream where the output will be printed to. - raw_ostream &getOutput() const; + void setOutput(std::unique_ptr<OutputStrategy> output); /// Print and clear the timing results. Only call this when there are no more /// references to nested timers around, as printing post-processes and clears @@ -408,6 +461,7 @@ protected: private: const std::unique_ptr<detail::DefaultTimingManagerImpl> impl; + std::unique_ptr<OutputStrategy> out; }; /// Register a set of useful command-line options that can be used to configure diff --git a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp index 0e3b646..25fa158 100644 --- a/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp +++ b/mlir/lib/Conversion/MemRefToEmitC/MemRefToEmitC.cpp @@ -62,8 +62,14 @@ struct ConvertLoad final : public OpConversionPattern<memref::LoadOp> { return rewriter.notifyMatchFailure(op.getLoc(), "cannot convert type"); } + auto arrayValue = + dyn_cast<TypedValue<emitc::ArrayType>>(operands.getMemref()); + if (!arrayValue) { + return rewriter.notifyMatchFailure(op.getLoc(), "expected array type"); + } + auto subscript = rewriter.create<emitc::SubscriptOp>( - op.getLoc(), operands.getMemref(), operands.getIndices()); + op.getLoc(), arrayValue, operands.getIndices()); auto noInit = emitc::OpaqueAttr::get(getContext(), ""); auto var = @@ -81,9 +87,14 @@ struct ConvertStore final : public OpConversionPattern<memref::StoreOp> { LogicalResult matchAndRewrite(memref::StoreOp op, OpAdaptor operands, ConversionPatternRewriter &rewriter) const override { + auto arrayValue = + dyn_cast<TypedValue<emitc::ArrayType>>(operands.getMemref()); + if (!arrayValue) { + return rewriter.notifyMatchFailure(op.getLoc(), "expected array type"); + } auto subscript = rewriter.create<emitc::SubscriptOp>( - op.getLoc(), operands.getMemref(), operands.getIndices()); + op.getLoc(), arrayValue, operands.getIndices()); rewriter.replaceOpWithNewOp<emitc::AssignOp>(op, subscript, operands.getValue()); return success(); diff --git a/mlir/lib/Dialect/Affine/Transforms/LoopCoalescing.cpp b/mlir/lib/Dialect/Affine/Transforms/LoopCoalescing.cpp index 1dc69ab..05c7707 100644 --- a/mlir/lib/Dialect/Affine/Transforms/LoopCoalescing.cpp +++ b/mlir/lib/Dialect/Affine/Transforms/LoopCoalescing.cpp @@ -39,9 +39,9 @@ struct LoopCoalescingPass func::FuncOp func = getOperation(); func.walk<WalkOrder::PreOrder>([](Operation *op) { if (auto scfForOp = dyn_cast<scf::ForOp>(op)) - (void)coalescePerfectlyNestedLoops(scfForOp); + (void)coalescePerfectlyNestedSCFForLoops(scfForOp); else if (auto affineForOp = dyn_cast<AffineForOp>(op)) - (void)coalescePerfectlyNestedLoops(affineForOp); + (void)coalescePerfectlyNestedAffineLoops(affineForOp); }); } }; diff --git a/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp b/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp index 37b36f7..117ee8e 100644 --- a/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp +++ b/mlir/lib/Dialect/Affine/Transforms/ReifyValueBounds.cpp @@ -84,7 +84,8 @@ FailureOr<OpFoldResult> mlir::affine::reifyShapedValueDimBound( OpBuilder &b, Location loc, presburger::BoundType type, Value value, int64_t dim, ValueBoundsConstraintSet::StopConditionFn stopCondition, bool closedUB) { - auto reifyToOperands = [&](Value v, std::optional<int64_t> d) { + auto reifyToOperands = [&](Value v, std::optional<int64_t> d, + ValueBoundsConstraintSet &cstr) { // We are trying to reify a bound for `value` in terms of the owning op's // operands. Construct a stop condition that evaluates to "true" for any SSA // value except for `value`. I.e., the bound will be computed in terms of @@ -100,7 +101,8 @@ FailureOr<OpFoldResult> mlir::affine::reifyShapedValueDimBound( FailureOr<OpFoldResult> mlir::affine::reifyIndexValueBound( OpBuilder &b, Location loc, presburger::BoundType type, Value value, ValueBoundsConstraintSet::StopConditionFn stopCondition, bool closedUB) { - auto reifyToOperands = [&](Value v, std::optional<int64_t> d) { + auto reifyToOperands = [&](Value v, std::optional<int64_t> d, + ValueBoundsConstraintSet &cstr) { return v != value; }; return reifyValueBound(b, loc, type, value, /*dim=*/std::nullopt, diff --git a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp index af59973..268050a 100644 --- a/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp +++ b/mlir/lib/Dialect/Affine/Utils/LoopUtils.cpp @@ -2765,3 +2765,51 @@ mlir::affine::separateFullTiles(MutableArrayRef<AffineForOp> inputNest, return success(); } + +LogicalResult affine::coalescePerfectlyNestedAffineLoops(AffineForOp op) { + LogicalResult result(failure()); + SmallVector<AffineForOp> loops; + getPerfectlyNestedLoops(loops, op); + if (loops.size() <= 1) + return success(); + + // Look for a band of loops that can be coalesced, i.e. perfectly nested + // loops with bounds defined above some loop. + // 1. For each loop, find above which parent loop its operands are + // defined. + SmallVector<unsigned> operandsDefinedAbove(loops.size()); + for (unsigned i = 0, e = loops.size(); i < e; ++i) { + operandsDefinedAbove[i] = i; + for (unsigned j = 0; j < i; ++j) { + if (areValuesDefinedAbove(loops[i].getOperands(), loops[j].getRegion())) { + operandsDefinedAbove[i] = j; + break; + } + } + } + + // 2. Identify bands of loops such that the operands of all of them are + // defined above the first loop in the band. Traverse the nest bottom-up + // so that modifications don't invalidate the inner loops. + for (unsigned end = loops.size(); end > 0; --end) { + unsigned start = 0; + for (; start < end - 1; ++start) { + auto maxPos = + *std::max_element(std::next(operandsDefinedAbove.begin(), start), + std::next(operandsDefinedAbove.begin(), end)); + if (maxPos > start) + continue; + assert(maxPos == start && + "expected loop bounds to be known at the start of the band"); + auto band = llvm::MutableArrayRef(loops.data() + start, end - start); + if (succeeded(coalesceLoops(band))) + result = success(); + break; + } + // If a band was found and transformed, keep looking at the loops above + // the outermost transformed loop. + if (start != end - 1) + end = start + 1; + } + return result; +} diff --git a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp index 8d9fd14..fad2212 100644 --- a/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp +++ b/mlir/lib/Dialect/Arith/Transforms/ReifyValueBounds.cpp @@ -119,7 +119,8 @@ FailureOr<OpFoldResult> mlir::arith::reifyShapedValueDimBound( OpBuilder &b, Location loc, presburger::BoundType type, Value value, int64_t dim, ValueBoundsConstraintSet::StopConditionFn stopCondition, bool closedUB) { - auto reifyToOperands = [&](Value v, std::optional<int64_t> d) { + auto reifyToOperands = [&](Value v, std::optional<int64_t> d, + ValueBoundsConstraintSet &cstr) { // We are trying to reify a bound for `value` in terms of the owning op's // operands. Construct a stop condition that evaluates to "true" for any SSA // value expect for `value`. I.e., the bound will be computed in terms of @@ -135,7 +136,8 @@ FailureOr<OpFoldResult> mlir::arith::reifyShapedValueDimBound( FailureOr<OpFoldResult> mlir::arith::reifyIndexValueBound( OpBuilder &b, Location loc, presburger::BoundType type, Value value, ValueBoundsConstraintSet::StopConditionFn stopCondition, bool closedUB) { - auto reifyToOperands = [&](Value v, std::optional<int64_t> d) { + auto reifyToOperands = [&](Value v, std::optional<int64_t> d, + ValueBoundsConstraintSet &cstr) { return v != value; }; return reifyValueBound(b, loc, type, value, /*dim=*/std::nullopt, diff --git a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp index 1f48d27..1374022 100644 --- a/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp +++ b/mlir/lib/Dialect/ArmNeon/Transforms/LowerContractionToSMMLAPattern.cpp @@ -40,8 +40,9 @@ static Type matchContainerType(Type element, Type container) { /// Lowering from a vector::contractOp arm neon smmla intrinsic. This will tile /// any vector.contract into multiple smmla instructions with unrolling so long -/// as [2,2,8] is a divisor of its shape. If no unrolling is necessary, a single -/// smmla instruction is emitted. +/// as [2,2,8] is a divisor of its shape. It can also process vecmats with dimM +/// = 1 (either explicitly or inferred if LHS has only dimK) If no unrolling is +/// necessary, a single smmla instruction is emitted. class LowerContractionToSMMLAPattern : public OpRewritePattern<vector::ContractionOp> { public: @@ -49,32 +50,35 @@ public: LogicalResult matchAndRewrite(vector::ContractionOp op, PatternRewriter &rewriter) const override { Location loc = op.getLoc(); - // Check index maps that represent M N K in contract. - auto indexingMaps = op.getIndexingMapsArray(); - if (llvm::any_of(indexingMaps, [](mlir::AffineMap affineMap) { - return affineMap.isPermutation() || affineMap.getNumDims() != 3 || - affineMap.getNumResults() != 2; - })) { - return failure(); - } - // Check iterator types for contract. - auto iteratorTypes = op.getIteratorTypesArray(); - if (iteratorTypes.size() != 3 || - iteratorTypes[0] != vector::IteratorType::parallel || - iteratorTypes[1] != vector::IteratorType::parallel || - iteratorTypes[2] != vector::IteratorType::reduction) { - return failure(); - } - // Infer tile sizes from operands; Note: RHS is not transposed. + // Infer tile sizes from operands. For vecmat, LHS may only have 1 dim. + // Note: RHS is not transposed. mlir::VectorType lhsType = op.getLhsType(); mlir::VectorType rhsType = op.getRhsType(); - auto dimM = lhsType.getDimSize(0); + auto dimM = lhsType.getRank() == 1 ? 1 : lhsType.getDimSize(0); auto dimN = rhsType.getDimSize(0); - auto dimK = lhsType.getDimSize(1); - + auto dimK = rhsType.getDimSize(1); + bool isVecmat = dimM == 1 ? true : false; + if (lhsType.getDimSize(lhsType.getRank() - 1) != + rhsType.getDimSize(rhsType.getRank() - 1)) { + return failure(); // dimK mismatch + } // Unrolling patterns can handle any [2, 2, 8] shaped multiple of inputs for // tiling. - if (dimM % 2 != 0 || dimN % 2 != 0 || dimK % 8 != 0) { + if ((dimM % 2 != 0 && !isVecmat) || dimN % 2 != 0 || dimK % 8 != 0) { + return failure(); + } + + // Check iterator types for contract. All iterators except inner-most + // dimension must be parallel. + auto iteratorTypes = op.getIteratorTypesArray(); + if (iteratorTypes.size() > 3 || iteratorTypes[iteratorTypes.size() - 1] != + vector::IteratorType::reduction) { + return failure(); + } + if (llvm::any_of(ArrayRef<vector::IteratorType>(iteratorTypes).drop_back(1), + [](vector::IteratorType iteratorType) { + return iteratorType != vector::IteratorType::parallel; + })) { return failure(); } @@ -120,11 +124,14 @@ public: loc, op.getResultType(), rewriter.getZeroAttr(op.getResultType())); SmallVector<int64_t> unrolledSize = *op.getShapeForUnroll(); - SmallVector<int64_t> smmlaShape{2, 2, 8}; - SmallVector<int64_t> loopOrder{0, 1, 2}; + SmallVector<int64_t> smmlaShape{2, 8}; + SmallVector<int64_t> loopOrder{0, 1}; + if (unrolledSize.size() == 3) { + smmlaShape.insert(smmlaShape.begin(), isVecmat ? 1 : 2); + loopOrder.push_back(2); + } for (SmallVector<int64_t> offsets : StaticTileOffsetRange(unrolledSize, smmlaShape, loopOrder)) { - // Helper to compute the new shape of each operand and extract the slice. auto extractOperand = [&](Value operand, AffineMap permutationMap, ArrayRef<int64_t> operandOffsets) { @@ -150,16 +157,40 @@ public: Value tiledAcc = extractOperand(op.getAcc(), accPermutationMap, accOffsets); + auto inputElementType = + tiledLhs.getType().cast<ShapedType>().getElementType(); + auto accElementType = + tiledAcc.getType().cast<ShapedType>().getElementType(); + auto inputExpandedType = VectorType::get({2, 8}, inputElementType); + auto outputExpandedType = VectorType::get({2, 2}, accElementType); + + // With vecmat, tiled LHS and ACC will contain only one of 2 necessary + // rows along dimM. Expand their shapes to match the smmla op. + if (isVecmat) { + auto expandForSMMLA = [&](Value tiledOperand, + VectorType expandedTypeType) { + auto emptyOperand = rewriter.create<arith::ConstantOp>( + loc, expandedTypeType, rewriter.getZeroAttr(expandedTypeType)); + SmallVector<int64_t> offsets( + emptyOperand.getType().cast<ShapedType>().getRank(), 0); + SmallVector<int64_t> strides( + tiledOperand.getType().cast<ShapedType>().getRank(), 1); + return rewriter.createOrFold<vector::InsertStridedSliceOp>( + loc, tiledOperand, emptyOperand, offsets, strides); + }; + tiledLhs = expandForSMMLA(tiledLhs, inputExpandedType); + tiledAcc = expandForSMMLA(tiledAcc, outputExpandedType); + } + // Collapse tiled operands to 1D vectors required by smmla intrinsic - auto collapsedInputType = VectorType::get( - tiledLhs.getType().cast<ShapedType>().getNumElements(), - tiledLhs.getType().cast<ShapedType>().getElementType()); - auto collapsedOutputType = VectorType::get( - {4}, tiledAcc.getType().cast<ShapedType>().getElementType()); + auto collapsedInputType = + VectorType::get(inputExpandedType.getNumElements(), inputElementType); auto collapsedLhs = rewriter.createOrFold<vector::ShapeCastOp>( tiledLhs.getLoc(), collapsedInputType, tiledLhs); auto collapsedRhs = rewriter.createOrFold<vector::ShapeCastOp>( tiledRhs.getLoc(), collapsedInputType, tiledRhs); + auto collapsedOutputType = + VectorType::get(outputExpandedType.getNumElements(), accElementType); auto collapsedRes = rewriter.createOrFold<vector::ShapeCastOp>( tiledAcc.getLoc(), collapsedOutputType, tiledAcc); @@ -172,6 +203,11 @@ public: Value tiledRes = rewriter.createOrFold<vector::ShapeCastOp>( smmlaOp.getLoc(), tiledAcc.getType(), smmlaOp); + // With vecmat, only one row of tiled ACC can be inserted inot file result + if (isVecmat) { + tiledRes = rewriter.createOrFold<vector::ExtractOp>(loc, tiledRes, 0); + } + // Insert the tiled result back into the non tiled result of the // contract op. SmallVector<int64_t> strides( diff --git a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp index 5d11f8f6..1320db3 100644 --- a/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp +++ b/mlir/lib/Dialect/ControlFlow/IR/ControlFlowOps.cpp @@ -531,8 +531,8 @@ static ParseResult parseSwitchOpCases( failed(parser.parseSuccessor(destination))) return failure(); if (succeeded(parser.parseOptionalLParen())) { - if (failed(parser.parseOperandList(operands, OpAsmParser::Delimiter::None, - /*allowResultNumber=*/false)) || + if (failed(parser.parseOperandList(operands, + OpAsmParser::Delimiter::None)) || failed(parser.parseColonTypeList(operandTypes)) || failed(parser.parseRParen())) return failure(); diff --git a/mlir/lib/Dialect/DLTI/DLTI.cpp b/mlir/lib/Dialect/DLTI/DLTI.cpp index daef234..98a8865 100644 --- a/mlir/lib/Dialect/DLTI/DLTI.cpp +++ b/mlir/lib/Dialect/DLTI/DLTI.cpp @@ -282,6 +282,11 @@ DataLayoutEntryListRef DataLayoutSpecAttr::getEntries() const { } StringAttr +DataLayoutSpecAttr::getEndiannessIdentifier(MLIRContext *context) const { + return Builder(context).getStringAttr(DLTIDialect::kDataLayoutEndiannessKey); +} + +StringAttr DataLayoutSpecAttr::getAllocaMemorySpaceIdentifier(MLIRContext *context) const { return Builder(context).getStringAttr( DLTIDialect::kDataLayoutAllocaMemorySpaceKey); diff --git a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp index f4a9dc3..7cbf28b 100644 --- a/mlir/lib/Dialect/EmitC/IR/EmitC.cpp +++ b/mlir/lib/Dialect/EmitC/IR/EmitC.cpp @@ -70,6 +70,11 @@ bool mlir::emitc::isSupportedIntegerType(Type type) { return false; } +bool mlir::emitc::isIntegerIndexOrOpaqueType(Type type) { + return llvm::isa<IndexType, emitc::OpaqueType>(type) || + isSupportedIntegerType(type); +} + bool mlir::emitc::isSupportedFloatType(Type type) { if (auto floatType = llvm::dyn_cast<FloatType>(type)) { switch (floatType.getWidth()) { @@ -780,12 +785,61 @@ LogicalResult emitc::YieldOp::verify() { //===----------------------------------------------------------------------===// LogicalResult emitc::SubscriptOp::verify() { - if (getIndices().size() != (size_t)getArray().getType().getRank()) { - return emitOpError() << "requires number of indices (" - << getIndices().size() - << ") to match the rank of the array type (" - << getArray().getType().getRank() << ")"; + // Checks for array operand. + if (auto arrayType = llvm::dyn_cast<emitc::ArrayType>(getValue().getType())) { + // Check number of indices. + if (getIndices().size() != (size_t)arrayType.getRank()) { + return emitOpError() << "on array operand requires number of indices (" + << getIndices().size() + << ") to match the rank of the array type (" + << arrayType.getRank() << ")"; + } + // Check types of index operands. + for (unsigned i = 0, e = getIndices().size(); i != e; ++i) { + Type type = getIndices()[i].getType(); + if (!isIntegerIndexOrOpaqueType(type)) { + return emitOpError() << "on array operand requires index operand " << i + << " to be integer-like, but got " << type; + } + } + // Check element type. + Type elementType = arrayType.getElementType(); + if (elementType != getType()) { + return emitOpError() << "on array operand requires element type (" + << elementType << ") and result type (" << getType() + << ") to match"; + } + return success(); } + + // Checks for pointer operand. + if (auto pointerType = + llvm::dyn_cast<emitc::PointerType>(getValue().getType())) { + // Check number of indices. + if (getIndices().size() != 1) { + return emitOpError() + << "on pointer operand requires one index operand, but got " + << getIndices().size(); + } + // Check types of index operand. + Type type = getIndices()[0].getType(); + if (!isIntegerIndexOrOpaqueType(type)) { + return emitOpError() << "on pointer operand requires index operand to be " + "integer-like, but got " + << type; + } + // Check pointee type. + Type pointeeType = pointerType.getPointee(); + if (pointeeType != getType()) { + return emitOpError() << "on pointer operand requires pointee type (" + << pointeeType << ") and result type (" << getType() + << ") to match"; + } + return success(); + } + + // The operand has opaque type, so we can't assume anything about the number + // or types of index operands. return success(); } diff --git a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp index 06c1fdd..5637552 100644 --- a/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp +++ b/mlir/lib/Dialect/LLVMIR/IR/LLVMMemorySlot.cpp @@ -168,13 +168,6 @@ bool LLVM::StoreOp::canUsesBeRemoved( DeletionKind LLVM::StoreOp::removeBlockingUses( const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses, RewriterBase &rewriter, Value reachingDefinition) { - // `canUsesBeRemoved` checked this blocking use must be the stored slot - // pointer. - for (Operation *user : slot.ptr.getUsers()) - if (auto declareOp = dyn_cast<LLVM::DbgDeclareOp>(user)) - rewriter.create<LLVM::DbgValueOp>(declareOp->getLoc(), getValue(), - declareOp.getVarInfo(), - declareOp.getLocationExpr()); return DeletionKind::Delete; } @@ -407,6 +400,18 @@ DeletionKind LLVM::DbgValueOp::removeBlockingUses( return DeletionKind::Keep; } +bool LLVM::DbgDeclareOp::requiresReplacedValues() { return true; } + +void LLVM::DbgDeclareOp::visitReplacedValues( + ArrayRef<std::pair<Operation *, Value>> definitions, + RewriterBase &rewriter) { + for (auto [op, value] : definitions) { + rewriter.setInsertionPointAfter(op); + rewriter.create<LLVM::DbgValueOp>(getLoc(), value, getVarInfo(), + getLocationExpr()); + } +} + //===----------------------------------------------------------------------===// // Interfaces for GEPOp //===----------------------------------------------------------------------===// diff --git a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp index 2d7219f..9c5c58f 100644 --- a/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp +++ b/mlir/lib/Dialect/Linalg/IR/LinalgOps.cpp @@ -373,14 +373,15 @@ namespace { class RegionBuilderHelper { public: - RegionBuilderHelper(MLIRContext *context, Block &block) - : context(context), block(block) {} + RegionBuilderHelper(OpBuilder &builder, Block &block) + : builder(builder), block(block) {} // Build the unary functions defined by OpDSL. Value buildUnaryFn(UnaryFn unaryFn, Value arg) { if (!isFloatingPoint(arg)) llvm_unreachable("unsupported non numeric type"); - OpBuilder builder = getBuilder(); + OpBuilder::InsertionGuard g(builder); + builder.setInsertionPointToEnd(&block); switch (unaryFn) { case UnaryFn::exp: return builder.create<math::ExpOp>(arg.getLoc(), arg); @@ -407,7 +408,8 @@ public: arg1.getType().getIntOrFloatBitWidth() == 1; if (!allComplex && !allFloatingPoint && !allInteger) llvm_unreachable("unsupported non numeric type"); - OpBuilder builder = getBuilder(); + OpBuilder::InsertionGuard g(builder); + builder.setInsertionPointToEnd(&block); switch (binaryFn) { case BinaryFn::add: if (allComplex) @@ -481,29 +483,32 @@ public: } void yieldOutputs(ValueRange values) { - OpBuilder builder = getBuilder(); + OpBuilder::InsertionGuard g(builder); + builder.setInsertionPointToEnd(&block); Location loc = builder.getUnknownLoc(); builder.create<YieldOp>(loc, values); } Value constant(const std::string &value) { - OpBuilder builder = getBuilder(); + OpBuilder::InsertionGuard g(builder); + builder.setInsertionPointToEnd(&block); Location loc = builder.getUnknownLoc(); Attribute valueAttr = parseAttribute(value, builder.getContext()); return builder.create<arith::ConstantOp>(loc, ::cast<TypedAttr>(valueAttr)); } Value index(int64_t dim) { - OpBuilder builder = getBuilder(); + OpBuilder::InsertionGuard g(builder); + builder.setInsertionPointToEnd(&block); return builder.create<IndexOp>(builder.getUnknownLoc(), dim); } Type getIntegerType(unsigned width) { - return IntegerType::get(context, width); + return IntegerType::get(builder.getContext(), width); } - Type getFloat32Type() { return Float32Type::get(context); } - Type getFloat64Type() { return Float64Type::get(context); } + Type getFloat32Type() { return Float32Type::get(builder.getContext()); } + Type getFloat64Type() { return Float64Type::get(builder.getContext()); } private: // Generates operations to cast the given operand to a specified type. @@ -511,7 +516,8 @@ private: // operand returned as-is (which will presumably yield a verification // issue downstream). Value cast(Type toType, Value operand, bool isUnsignedCast) { - OpBuilder builder = getBuilder(); + OpBuilder::InsertionGuard g(builder); + builder.setInsertionPointToEnd(&block); auto loc = operand.getLoc(); return convertScalarToDtype(builder, loc, operand, toType, isUnsignedCast); } @@ -526,13 +532,7 @@ private: return llvm::isa<IntegerType>(value.getType()); } - OpBuilder getBuilder() { - OpBuilder builder(context); - builder.setInsertionPointToEnd(&block); - return builder; - } - - MLIRContext *context; + OpBuilder &builder; Block █ }; diff --git a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp index 58fb2e9..899b8c8 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/BufferizableOpInterfaceImpl.cpp @@ -11,6 +11,7 @@ #include "mlir/Dialect/Bufferization/IR/Bufferization.h" #include "mlir/Dialect/Bufferization/IR/DstBufferizableOpInterfaceImpl.h" #include "mlir/Dialect/Linalg/IR/Linalg.h" +#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h" #include "mlir/Dialect/Tensor/IR/Tensor.h" #include "mlir/IR/Dialect.h" #include "mlir/IR/Operation.h" @@ -110,6 +111,10 @@ struct LinalgOpInterface ArrayRef<OpOperand *> opOperands) const { auto linalgOp = cast<linalg::LinalgOp>(op); + // Accesses into sparse data structures are not necessarily elementwise. + if (sparse_tensor::hasAnySparseOperand(linalgOp)) + return false; + // All loops must be parallel. if (linalgOp.getNumLoops() != linalgOp.getNumParallelLoops()) return false; diff --git a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp index b32ea8e..c3a08ce 100644 --- a/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp +++ b/mlir/lib/Dialect/Linalg/Transforms/HoistPadding.cpp @@ -468,7 +468,7 @@ HoistPaddingAnalysis::getHoistedPackedTensorSizes(RewriterBase &rewriter, FailureOr<OpFoldResult> loopUb = affine::reifyIndexValueBound( rewriter, loc, presburger::BoundType::UB, forOp.getUpperBound(), /*stopCondition=*/ - [&](Value v, std::optional<int64_t> d) { + [&](Value v, std::optional<int64_t> d, ValueBoundsConstraintSet &cstr) { if (v == forOp.getUpperBound()) return false; // Compute a bound that is independent of any affine op results. diff --git a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp index 0b85462..42629e1 100644 --- a/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp +++ b/mlir/lib/Dialect/Math/Transforms/ExpandPatterns.cpp @@ -216,20 +216,30 @@ static LogicalResult convertCeilOp(math::CeilOp op, PatternRewriter &rewriter) { // Convert `math.fpowi` to a series of `arith.mulf` operations. // If the power is negative, we divide one by the result. // If both the base and power are zero, the result is 1. -static LogicalResult convertFPowICstOp(math::FPowIOp op, - PatternRewriter &rewriter) { +// In the case of non constant power, we convert the operation to `math.powf`. +static LogicalResult convertFPowIOp(math::FPowIOp op, + PatternRewriter &rewriter) { ImplicitLocOpBuilder b(op->getLoc(), rewriter); Value base = op.getOperand(0); Value power = op.getOperand(1); Type baseType = base.getType(); + auto convertFPowItoPowf = [&]() -> LogicalResult { + Value castPowerToFp = + rewriter.create<arith::SIToFPOp>(op.getLoc(), baseType, power); + Value res = rewriter.create<math::PowFOp>(op.getLoc(), baseType, base, + castPowerToFp); + rewriter.replaceOp(op, res); + return success(); + }; + Attribute cstAttr; if (!matchPattern(power, m_Constant(&cstAttr))) - return failure(); + return convertFPowItoPowf(); APInt value; if (!matchPattern(cstAttr, m_ConstantInt(&value))) - return failure(); + return convertFPowItoPowf(); int64_t powerInt = value.getSExtValue(); bool isNegative = powerInt < 0; @@ -591,7 +601,7 @@ void mlir::populateExpandPowFPattern(RewritePatternSet &patterns) { } void mlir::populateExpandFPowIPattern(RewritePatternSet &patterns) { - patterns.add(convertFPowICstOp); + patterns.add(convertFPowIOp); } void mlir::populateExpandRoundFPattern(RewritePatternSet &patterns) { diff --git a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp index bf58750..a043431 100644 --- a/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp +++ b/mlir/lib/Dialect/OpenMP/IR/OpenMPDialect.cpp @@ -1538,6 +1538,21 @@ static void printAtomicReductionRegion(OpAsmPrinter &printer, printer.printRegion(region); } +static ParseResult parseCleanupReductionRegion(OpAsmParser &parser, + Region ®ion) { + if (parser.parseOptionalKeyword("cleanup")) + return success(); + return parser.parseRegion(region); +} + +static void printCleanupReductionRegion(OpAsmPrinter &printer, + DeclareReductionOp op, Region ®ion) { + if (region.empty()) + return; + printer << "cleanup "; + printer.printRegion(region); +} + LogicalResult DeclareReductionOp::verifyRegions() { if (getInitializerRegion().empty()) return emitOpError() << "expects non-empty initializer region"; @@ -1571,21 +1586,29 @@ LogicalResult DeclareReductionOp::verifyRegions() { "of the reduction type"; } - if (getAtomicReductionRegion().empty()) + if (!getAtomicReductionRegion().empty()) { + Block &atomicReductionEntryBlock = getAtomicReductionRegion().front(); + if (atomicReductionEntryBlock.getNumArguments() != 2 || + atomicReductionEntryBlock.getArgumentTypes()[0] != + atomicReductionEntryBlock.getArgumentTypes()[1]) + return emitOpError() << "expects atomic reduction region with two " + "arguments of the same type"; + auto ptrType = llvm::dyn_cast<PointerLikeType>( + atomicReductionEntryBlock.getArgumentTypes()[0]); + if (!ptrType || + (ptrType.getElementType() && ptrType.getElementType() != getType())) + return emitOpError() << "expects atomic reduction region arguments to " + "be accumulators containing the reduction type"; + } + + if (getCleanupRegion().empty()) return success(); + Block &cleanupEntryBlock = getCleanupRegion().front(); + if (cleanupEntryBlock.getNumArguments() != 1 || + cleanupEntryBlock.getArgument(0).getType() != getType()) + return emitOpError() << "expects cleanup region with one argument " + "of the reduction type"; - Block &atomicReductionEntryBlock = getAtomicReductionRegion().front(); - if (atomicReductionEntryBlock.getNumArguments() != 2 || - atomicReductionEntryBlock.getArgumentTypes()[0] != - atomicReductionEntryBlock.getArgumentTypes()[1]) - return emitOpError() << "expects atomic reduction region with two " - "arguments of the same type"; - auto ptrType = llvm::dyn_cast<PointerLikeType>( - atomicReductionEntryBlock.getArgumentTypes()[0]); - if (!ptrType || - (ptrType.getElementType() && ptrType.getElementType() != getType())) - return emitOpError() << "expects atomic reduction region arguments to " - "be accumulators containing the reduction type"; return success(); } diff --git a/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp b/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp index cb36e0c..1e13e60 100644 --- a/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp +++ b/mlir/lib/Dialect/SCF/IR/ValueBoundsOpInterfaceImpl.cpp @@ -58,7 +58,7 @@ struct ForOpInterface ValueDimList boundOperands; LogicalResult status = ValueBoundsConstraintSet::computeBound( bound, boundOperands, BoundType::EQ, yieldedValue, dim, - [&](Value v, std::optional<int64_t> d) { + [&](Value v, std::optional<int64_t> d, ValueBoundsConstraintSet &cstr) { // Stop when reaching a block argument of the loop body. if (auto bbArg = llvm::dyn_cast<BlockArgument>(v)) return bbArg.getOwner()->getParentOp() == forOp; diff --git a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp index c091841..7e4faf8 100644 --- a/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp +++ b/mlir/lib/Dialect/SCF/TransformOps/SCFTransformOps.cpp @@ -332,9 +332,9 @@ transform::LoopCoalesceOp::applyToOne(transform::TransformRewriter &rewriter, transform::TransformState &state) { LogicalResult result(failure()); if (scf::ForOp scfForOp = dyn_cast<scf::ForOp>(op)) - result = coalescePerfectlyNestedLoops(scfForOp); + result = coalescePerfectlyNestedSCFForLoops(scfForOp); else if (AffineForOp affineForOp = dyn_cast<AffineForOp>(op)) - result = coalescePerfectlyNestedLoops(affineForOp); + result = coalescePerfectlyNestedAffineLoops(affineForOp); results.push_back(op); if (failed(result)) { diff --git a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopCollapsing.cpp b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopCollapsing.cpp index a69df02..6ba7020 100644 --- a/mlir/lib/Dialect/SCF/Transforms/ParallelLoopCollapsing.cpp +++ b/mlir/lib/Dialect/SCF/Transforms/ParallelLoopCollapsing.cpp @@ -28,6 +28,7 @@ namespace { struct TestSCFParallelLoopCollapsing : public impl::TestSCFParallelLoopCollapsingBase< TestSCFParallelLoopCollapsing> { + void runOnOperation() override { Operation *module = getOperation(); @@ -88,6 +89,7 @@ struct TestSCFParallelLoopCollapsing // Only apply the transformation on parallel loops where the specified // transformation is valid, but do NOT early abort in the case of invalid // loops. + IRRewriter rewriter(&getContext()); module->walk([&](scf::ParallelOp op) { if (flattenedCombinedLoops.size() != op.getNumLoops()) { op.emitOpError("has ") @@ -97,7 +99,7 @@ struct TestSCFParallelLoopCollapsing << flattenedCombinedLoops.size() << " iter args."; return; } - collapseParallelLoops(op, combinedLoops); + collapseParallelLoops(rewriter, op, combinedLoops); }); } }; diff --git a/mlir/lib/Dialect/SCF/Utils/Utils.cpp b/mlir/lib/Dialect/SCF/Utils/Utils.cpp index 914aeb4..9279081 100644 --- a/mlir/lib/Dialect/SCF/Utils/Utils.cpp +++ b/mlir/lib/Dialect/SCF/Utils/Utils.cpp @@ -13,6 +13,7 @@ #include "mlir/Dialect/SCF/Utils/Utils.h" #include "mlir/Analysis/SliceAnalysis.h" #include "mlir/Dialect/Arith/IR/Arith.h" +#include "mlir/Dialect/Arith/Utils/Utils.h" #include "mlir/Dialect/Func/IR/FuncOps.h" #include "mlir/Dialect/SCF/IR/SCF.h" #include "mlir/IR/BuiltinOps.h" @@ -472,18 +473,23 @@ LogicalResult mlir::loopUnrollByFactor( return success(); } -/// Return the new lower bound, upper bound, and step in that order. Insert any -/// additional bounds calculations before the given builder and any additional -/// conversion back to the original loop induction value inside the given Block. -static LoopParams normalizeLoop(OpBuilder &boundsBuilder, - OpBuilder &insideLoopBuilder, Location loc, - Value lowerBound, Value upperBound, Value step, - Value inductionVar) { +/// Transform a loop with a strictly positive step +/// for %i = %lb to %ub step %s +/// into a 0-based loop with step 1 +/// for %ii = 0 to ceildiv(%ub - %lb, %s) step 1 { +/// %i = %ii * %s + %lb +/// Insert the induction variable remapping in the body of `inner`, which is +/// expected to be either `loop` or another loop perfectly nested under `loop`. +/// Insert the definition of new bounds immediate before `outer`, which is +/// expected to be either `loop` or its parent in the loop nest. +static LoopParams emitNormalizedLoopBounds(RewriterBase &rewriter, Location loc, + Value lb, Value ub, Value step) { + // For non-index types, generate `arith` instructions // Check if the loop is already known to have a constant zero lower bound or // a constant one step. bool isZeroBased = false; - if (auto ubCst = getConstantIntValue(lowerBound)) - isZeroBased = ubCst.value() == 0; + if (auto lbCst = getConstantIntValue(lb)) + isZeroBased = lbCst.value() == 0; bool isStepOne = false; if (auto stepCst = getConstantIntValue(step)) @@ -493,62 +499,90 @@ static LoopParams normalizeLoop(OpBuilder &boundsBuilder, // assuming the step is strictly positive. Update the bounds and the step // of the loop to go from 0 to the number of iterations, if necessary. if (isZeroBased && isStepOne) - return {/*lowerBound=*/lowerBound, /*upperBound=*/upperBound, - /*step=*/step}; + return {lb, ub, step}; - Value diff = boundsBuilder.create<arith::SubIOp>(loc, upperBound, lowerBound); + Value diff = isZeroBased ? ub : rewriter.create<arith::SubIOp>(loc, ub, lb); Value newUpperBound = - boundsBuilder.create<arith::CeilDivSIOp>(loc, diff, step); - - Value newLowerBound = - isZeroBased ? lowerBound - : boundsBuilder.create<arith::ConstantOp>( - loc, boundsBuilder.getZeroAttr(lowerBound.getType())); - Value newStep = - isStepOne ? step - : boundsBuilder.create<arith::ConstantOp>( - loc, boundsBuilder.getIntegerAttr(step.getType(), 1)); - - // Insert code computing the value of the original loop induction variable - // from the "normalized" one. - Value scaled = - isStepOne - ? inductionVar - : insideLoopBuilder.create<arith::MulIOp>(loc, inductionVar, step); - Value shifted = - isZeroBased - ? scaled - : insideLoopBuilder.create<arith::AddIOp>(loc, scaled, lowerBound); - - SmallPtrSet<Operation *, 2> preserve{scaled.getDefiningOp(), - shifted.getDefiningOp()}; - inductionVar.replaceAllUsesExcept(shifted, preserve); - return {/*lowerBound=*/newLowerBound, /*upperBound=*/newUpperBound, - /*step=*/newStep}; + isStepOne ? diff : rewriter.create<arith::CeilDivSIOp>(loc, diff, step); + + Value newLowerBound = isZeroBased + ? lb + : rewriter.create<arith::ConstantOp>( + loc, rewriter.getZeroAttr(lb.getType())); + Value newStep = isStepOne + ? step + : rewriter.create<arith::ConstantOp>( + loc, rewriter.getIntegerAttr(step.getType(), 1)); + + return {newLowerBound, newUpperBound, newStep}; } -/// Transform a loop with a strictly positive step -/// for %i = %lb to %ub step %s -/// into a 0-based loop with step 1 -/// for %ii = 0 to ceildiv(%ub - %lb, %s) step 1 { -/// %i = %ii * %s + %lb -/// Insert the induction variable remapping in the body of `inner`, which is -/// expected to be either `loop` or another loop perfectly nested under `loop`. -/// Insert the definition of new bounds immediate before `outer`, which is -/// expected to be either `loop` or its parent in the loop nest. -static void normalizeLoop(scf::ForOp loop, scf::ForOp outer, scf::ForOp inner) { - OpBuilder builder(outer); - OpBuilder innerBuilder = OpBuilder::atBlockBegin(inner.getBody()); - auto loopPieces = normalizeLoop(builder, innerBuilder, loop.getLoc(), - loop.getLowerBound(), loop.getUpperBound(), - loop.getStep(), loop.getInductionVar()); - - loop.setLowerBound(loopPieces.lowerBound); - loop.setUpperBound(loopPieces.upperBound); - loop.setStep(loopPieces.step); +/// Get back the original induction variable values after loop normalization +static void denormalizeInductionVariable(RewriterBase &rewriter, Location loc, + Value normalizedIv, Value origLb, + Value origStep) { + Value denormalizedIv; + SmallPtrSet<Operation *, 2> preserve; + bool isStepOne = isConstantIntValue(origStep, 1); + bool isZeroBased = isConstantIntValue(origLb, 0); + + Value scaled = normalizedIv; + if (!isStepOne) { + scaled = rewriter.create<arith::MulIOp>(loc, normalizedIv, origStep); + preserve.insert(scaled.getDefiningOp()); + } + denormalizedIv = scaled; + if (!isZeroBased) { + denormalizedIv = rewriter.create<arith::AddIOp>(loc, scaled, origLb); + preserve.insert(denormalizedIv.getDefiningOp()); + } + + rewriter.replaceAllUsesExcept(normalizedIv, denormalizedIv, preserve); } -LogicalResult mlir::coalesceLoops(MutableArrayRef<scf::ForOp> loops) { +/// Helper function to multiply a sequence of values. +static Value getProductOfIntsOrIndexes(RewriterBase &rewriter, Location loc, + ArrayRef<Value> values) { + assert(!values.empty() && "unexpected empty list"); + Value productOf = values.front(); + for (auto v : values.drop_front()) { + productOf = rewriter.create<arith::MulIOp>(loc, productOf, v); + } + return productOf; +} + +/// For each original loop, the value of the +/// induction variable can be obtained by dividing the induction variable of +/// the linearized loop by the total number of iterations of the loops nested +/// in it modulo the number of iterations in this loop (remove the values +/// related to the outer loops): +/// iv_i = floordiv(iv_linear, product-of-loop-ranges-until-i) mod range_i. +/// Compute these iteratively from the innermost loop by creating a "running +/// quotient" of division by the range. +static std::pair<SmallVector<Value>, SmallPtrSet<Operation *, 2>> +delinearizeInductionVariable(RewriterBase &rewriter, Location loc, + Value linearizedIv, ArrayRef<Value> ubs) { + Value previous = linearizedIv; + SmallVector<Value> delinearizedIvs(ubs.size()); + SmallPtrSet<Operation *, 2> preservedUsers; + for (unsigned i = 0, e = ubs.size(); i < e; ++i) { + unsigned idx = ubs.size() - i - 1; + if (i != 0) { + previous = rewriter.create<arith::DivSIOp>(loc, previous, ubs[idx + 1]); + preservedUsers.insert(previous.getDefiningOp()); + } + Value iv = previous; + if (i != e - 1) { + iv = rewriter.create<arith::RemSIOp>(loc, previous, ubs[idx]); + preservedUsers.insert(iv.getDefiningOp()); + } + delinearizedIvs[idx] = iv; + } + return {delinearizedIvs, preservedUsers}; +} + +LogicalResult mlir::coalesceLoops(RewriterBase &rewriter, + MutableArrayRef<scf::ForOp> loops) { if (loops.size() < 2) return failure(); @@ -557,57 +591,148 @@ LogicalResult mlir::coalesceLoops(MutableArrayRef<scf::ForOp> loops) { // 1. Make sure all loops iterate from 0 to upperBound with step 1. This // allows the following code to assume upperBound is the number of iterations. - for (auto loop : loops) - normalizeLoop(loop, outermost, innermost); + for (auto loop : loops) { + OpBuilder::InsertionGuard g(rewriter); + rewriter.setInsertionPoint(outermost); + Value lb = loop.getLowerBound(); + Value ub = loop.getUpperBound(); + Value step = loop.getStep(); + auto newLoopParams = + emitNormalizedLoopBounds(rewriter, loop.getLoc(), lb, ub, step); + + rewriter.modifyOpInPlace(loop, [&]() { + loop.setLowerBound(newLoopParams.lowerBound); + loop.setUpperBound(newLoopParams.upperBound); + loop.setStep(newLoopParams.step); + }); + + rewriter.setInsertionPointToStart(innermost.getBody()); + denormalizeInductionVariable(rewriter, loop.getLoc(), + loop.getInductionVar(), lb, step); + } // 2. Emit code computing the upper bound of the coalesced loop as product // of the number of iterations of all loops. - OpBuilder builder(outermost); + OpBuilder::InsertionGuard g(rewriter); + rewriter.setInsertionPoint(outermost); Location loc = outermost.getLoc(); - Value upperBound = outermost.getUpperBound(); - for (auto loop : loops.drop_front()) - upperBound = - builder.create<arith::MulIOp>(loc, upperBound, loop.getUpperBound()); + SmallVector<Value> upperBounds = llvm::map_to_vector( + loops, [](auto loop) { return loop.getUpperBound(); }); + Value upperBound = getProductOfIntsOrIndexes(rewriter, loc, upperBounds); outermost.setUpperBound(upperBound); - builder.setInsertionPointToStart(outermost.getBody()); - - // 3. Remap induction variables. For each original loop, the value of the - // induction variable can be obtained by dividing the induction variable of - // the linearized loop by the total number of iterations of the loops nested - // in it modulo the number of iterations in this loop (remove the values - // related to the outer loops): - // iv_i = floordiv(iv_linear, product-of-loop-ranges-until-i) mod range_i. - // Compute these iteratively from the innermost loop by creating a "running - // quotient" of division by the range. - Value previous = outermost.getInductionVar(); + rewriter.setInsertionPointToStart(innermost.getBody()); + auto [delinearizeIvs, preservedUsers] = delinearizeInductionVariable( + rewriter, loc, outermost.getInductionVar(), upperBounds); + rewriter.replaceAllUsesExcept(outermost.getInductionVar(), delinearizeIvs[0], + preservedUsers); + + for (int i = loops.size() - 1; i > 0; --i) { + auto outerLoop = loops[i - 1]; + auto innerLoop = loops[i]; + + Operation *innerTerminator = innerLoop.getBody()->getTerminator(); + auto yieldedVals = llvm::to_vector(innerTerminator->getOperands()); + rewriter.eraseOp(innerTerminator); + + SmallVector<Value> innerBlockArgs; + innerBlockArgs.push_back(delinearizeIvs[i]); + llvm::append_range(innerBlockArgs, outerLoop.getRegionIterArgs()); + rewriter.inlineBlockBefore(innerLoop.getBody(), outerLoop.getBody(), + Block::iterator(innerLoop), innerBlockArgs); + rewriter.replaceOp(innerLoop, yieldedVals); + } + return success(); +} + +LogicalResult mlir::coalesceLoops(MutableArrayRef<scf::ForOp> loops) { + if (loops.empty()) { + return failure(); + } + IRRewriter rewriter(loops.front().getContext()); + return coalesceLoops(rewriter, loops); +} + +LogicalResult mlir::coalescePerfectlyNestedSCFForLoops(scf::ForOp op) { + LogicalResult result(failure()); + SmallVector<scf::ForOp> loops; + getPerfectlyNestedLoops(loops, op); + + // Look for a band of loops that can be coalesced, i.e. perfectly nested + // loops with bounds defined above some loop. + + // 1. For each loop, find above which parent loop its bounds operands are + // defined. + SmallVector<unsigned> operandsDefinedAbove(loops.size()); for (unsigned i = 0, e = loops.size(); i < e; ++i) { - unsigned idx = loops.size() - i - 1; - if (i != 0) - previous = builder.create<arith::DivSIOp>(loc, previous, - loops[idx + 1].getUpperBound()); - - Value iv = (i == e - 1) ? previous - : builder.create<arith::RemSIOp>( - loc, previous, loops[idx].getUpperBound()); - replaceAllUsesInRegionWith(loops[idx].getInductionVar(), iv, - loops.back().getRegion()); + operandsDefinedAbove[i] = i; + for (unsigned j = 0; j < i; ++j) { + SmallVector<Value> boundsOperands = {loops[i].getLowerBound(), + loops[i].getUpperBound(), + loops[i].getStep()}; + if (areValuesDefinedAbove(boundsOperands, loops[j].getRegion())) { + operandsDefinedAbove[i] = j; + break; + } + } } - // 4. Move the operations from the innermost just above the second-outermost - // loop, delete the extra terminator and the second-outermost loop. - scf::ForOp second = loops[1]; - innermost.getBody()->back().erase(); - outermost.getBody()->getOperations().splice( - Block::iterator(second.getOperation()), - innermost.getBody()->getOperations()); - second.erase(); - return success(); + // 2. For each inner loop check that the iter_args for the immediately outer + // loop are the init for the immediately inner loop and that the yields of the + // return of the inner loop is the yield for the immediately outer loop. Keep + // track of where the chain starts from for each loop. + SmallVector<unsigned> iterArgChainStart(loops.size()); + iterArgChainStart[0] = 0; + for (unsigned i = 1, e = loops.size(); i < e; ++i) { + // By default set the start of the chain to itself. + iterArgChainStart[i] = i; + auto outerloop = loops[i - 1]; + auto innerLoop = loops[i]; + if (outerloop.getNumRegionIterArgs() != innerLoop.getNumRegionIterArgs()) { + continue; + } + if (!llvm::equal(outerloop.getRegionIterArgs(), innerLoop.getInitArgs())) { + continue; + } + auto outerloopTerminator = outerloop.getBody()->getTerminator(); + if (!llvm::equal(outerloopTerminator->getOperands(), + innerLoop.getResults())) { + continue; + } + iterArgChainStart[i] = iterArgChainStart[i - 1]; + } + + // 3. Identify bands of loops such that the operands of all of them are + // defined above the first loop in the band. Traverse the nest bottom-up + // so that modifications don't invalidate the inner loops. + for (unsigned end = loops.size(); end > 0; --end) { + unsigned start = 0; + for (; start < end - 1; ++start) { + auto maxPos = + *std::max_element(std::next(operandsDefinedAbove.begin(), start), + std::next(operandsDefinedAbove.begin(), end)); + if (maxPos > start) + continue; + if (iterArgChainStart[end - 1] > start) + continue; + auto band = llvm::MutableArrayRef(loops.data() + start, end - start); + if (succeeded(coalesceLoops(band))) + result = success(); + break; + } + // If a band was found and transformed, keep looking at the loops above + // the outermost transformed loop. + if (start != end - 1) + end = start + 1; + } + return result; } void mlir::collapseParallelLoops( - scf::ParallelOp loops, ArrayRef<std::vector<unsigned>> combinedDimensions) { - OpBuilder outsideBuilder(loops); + RewriterBase &rewriter, scf::ParallelOp loops, + ArrayRef<std::vector<unsigned>> combinedDimensions) { + OpBuilder::InsertionGuard g(rewriter); + rewriter.setInsertionPoint(loops); Location loc = loops.getLoc(); // Presort combined dimensions. @@ -619,25 +744,29 @@ void mlir::collapseParallelLoops( SmallVector<Value, 3> normalizedLowerBounds, normalizedSteps, normalizedUpperBounds; for (unsigned i = 0, e = loops.getNumLoops(); i < e; ++i) { - OpBuilder insideLoopBuilder = OpBuilder::atBlockBegin(loops.getBody()); - auto resultBounds = - normalizeLoop(outsideBuilder, insideLoopBuilder, loc, - loops.getLowerBound()[i], loops.getUpperBound()[i], - loops.getStep()[i], loops.getBody()->getArgument(i)); - - normalizedLowerBounds.push_back(resultBounds.lowerBound); - normalizedUpperBounds.push_back(resultBounds.upperBound); - normalizedSteps.push_back(resultBounds.step); + OpBuilder::InsertionGuard g2(rewriter); + rewriter.setInsertionPoint(loops); + Value lb = loops.getLowerBound()[i]; + Value ub = loops.getUpperBound()[i]; + Value step = loops.getStep()[i]; + auto newLoopParams = emitNormalizedLoopBounds(rewriter, loc, lb, ub, step); + normalizedLowerBounds.push_back(newLoopParams.lowerBound); + normalizedUpperBounds.push_back(newLoopParams.upperBound); + normalizedSteps.push_back(newLoopParams.step); + + rewriter.setInsertionPointToStart(loops.getBody()); + denormalizeInductionVariable(rewriter, loc, loops.getInductionVars()[i], lb, + step); } // Combine iteration spaces. SmallVector<Value, 3> lowerBounds, upperBounds, steps; - auto cst0 = outsideBuilder.create<arith::ConstantIndexOp>(loc, 0); - auto cst1 = outsideBuilder.create<arith::ConstantIndexOp>(loc, 1); + auto cst0 = rewriter.create<arith::ConstantIndexOp>(loc, 0); + auto cst1 = rewriter.create<arith::ConstantIndexOp>(loc, 1); for (auto &sortedDimension : sortedDimensions) { - Value newUpperBound = outsideBuilder.create<arith::ConstantIndexOp>(loc, 1); + Value newUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, 1); for (auto idx : sortedDimension) { - newUpperBound = outsideBuilder.create<arith::MulIOp>( + newUpperBound = rewriter.create<arith::MulIOp>( loc, newUpperBound, normalizedUpperBounds[idx]); } lowerBounds.push_back(cst0); @@ -651,7 +780,7 @@ void mlir::collapseParallelLoops( // value. The remainders then determine based on that range, which iteration // of the original induction value this represents. This is a normalized value // that is un-normalized already by the previous logic. - auto newPloop = outsideBuilder.create<scf::ParallelOp>( + auto newPloop = rewriter.create<scf::ParallelOp>( loc, lowerBounds, upperBounds, steps, [&](OpBuilder &insideBuilder, Location, ValueRange ploopIVs) { for (unsigned i = 0, e = combinedDimensions.size(); i < e; ++i) { diff --git a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorPasses.cpp b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorPasses.cpp index d4c1792..acea25f 100644 --- a/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorPasses.cpp +++ b/mlir/lib/Dialect/SparseTensor/Transforms/SparseTensorPasses.cpp @@ -274,7 +274,7 @@ struct SparseTensorCodegenPass }); // The following operations and dialects may be introduced by the // codegen rules, and are therefore marked as legal. - target.addLegalOp<linalg::FillOp>(); + target.addLegalOp<linalg::FillOp, linalg::YieldOp>(); target.addLegalDialect< arith::ArithDialect, bufferization::BufferizationDialect, complex::ComplexDialect, memref::MemRefDialect, scf::SCFDialect>(); diff --git a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp index 6e6e843..e06ac9a 100644 --- a/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp +++ b/mlir/lib/Dialect/Tosa/IR/TosaOps.cpp @@ -955,25 +955,34 @@ LogicalResult tosa::ReshapeOp::inferReturnTypeComponents( } mlir::LogicalResult tosa::ReshapeOp::verify() { - ShapedType inputType = llvm::cast<ShapedType>(getInput1().getType()); - ShapedType outputType = llvm::cast<ShapedType>(getType()); + TensorType inputType = getInput1().getType(); + RankedTensorType outputType = getType(); if (hasZeroDimension(inputType) || hasZeroDimension(outputType)) return emitOpError() << "tensor has a dimension with size zero. Each " "dimension of a tensor must have size >= 1"; + if ((int64_t) getNewShape().size() != outputType.getRank()) + return emitOpError() << "new shape does not match result rank"; + + for (auto [newShapeDim, outputShapeDim] : + zip(getNewShape(), outputType.getShape())) + if (newShapeDim != -1 && outputShapeDim != ShapedType::kDynamic && + newShapeDim != outputShapeDim) + return emitOpError() << "new shape is inconsistent with result shape"; + if (inputType.hasStaticShape() && outputType.hasStaticShape()) { int64_t inputElementsNum = inputType.getNumElements(); int64_t outputElementsNum = outputType.getNumElements(); if (inputElementsNum != outputElementsNum) { - return emitOpError() << "Cannot reshape " << inputElementsNum + return emitOpError() << "cannot reshape " << inputElementsNum << " elements into " << outputElementsNum; } } int missingDims = llvm::count(getNewShape(), -1); if (missingDims > 1) - return emitOpError() << "At most one target dimension can be -1"; + return emitOpError() << "expected at most one target dimension to be -1"; return mlir::success(); } diff --git a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp index ad28c56..8614559 100644 --- a/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp +++ b/mlir/lib/Dialect/Tosa/Transforms/TosaInferShapes.cpp @@ -18,14 +18,9 @@ #include "mlir/Dialect/Tosa/IR/TosaOps.h" #include "mlir/Dialect/Tosa/Utils/ShapeUtils.h" #include "mlir/IR/Builders.h" -#include "mlir/IR/BuiltinOps.h" -#include "mlir/IR/IRMapping.h" -#include "mlir/IR/Matchers.h" #include "mlir/Interfaces/InferTypeOpInterface.h" #include "mlir/Pass/Pass.h" #include "mlir/Transforms/DialectConversion.h" -#include "mlir/Transforms/GreedyPatternRewriteDriver.h" -#include "llvm/Support/FormatVariadic.h" namespace mlir { namespace tosa { @@ -39,9 +34,87 @@ using namespace mlir::tosa; namespace { -void propagateShapesInRegion(Region ®ion); +// Check whether this use case is replaceable. We define an op as +// being replaceable if it is used by a TosaOp, or an op with a +// type-inference related interface. +// When a non-replaceable use is encountered, the value is wrapped in a +// cast back to the original type after inference. +bool isReplaceableUser(Operation *user) { + // Handle unregistered dialects. + if (!user->getDialect()) + return false; + + return user->getDialect()->getNamespace() == + TosaDialect::getDialectNamespace() || + isa<InferTypeOpInterface, InferShapedTypeOpInterface>(user); +} + +// During type propagation, the types of values in the operator graph are +// updated. For the tosa.while_loop operation, types are speculatively updated +// within the body region to determine the output type of the while_loop. This +// process is performed until a fixed point is reached, then the types are +// reverted. +// +// This class encapsulates the state information needed to perform the reversion +// process or to commit to the final changes. +class TypeModificationState { +public: + TypeModificationState() = default; + + ~TypeModificationState() { + // Ensure the recorded modifications are either committed or reverted. + assert(oldTypes.empty() && "unhandled type modifications"); + } + + // Update the state of the value and record the old type. + void setType(Value value, Type type) { + if (value.getType() != type) { + oldTypes.emplace_back(value, value.getType()); + value.setType(type); + } + } -void propagateShapesToTosaIf(Operation &op) { + // Revert changes made to the types in the IR by setting all the affected + // values to their old types. + void revert() { + // Otherwise revert the changes. + for (auto [value, type] : oldTypes) + value.setType(type); + + oldTypes.clear(); + } + + // Commit the changes to the types in the IR. + // This requires inserting tensor.cast operations to mediate the newly + // inferred result types with users that do not support type inference. + void commit() { + // For each use whose type changed, cast the value with the new type back to + // the old type. + for (auto [value, oldType] : oldTypes) { + for (auto &use : value.getUses()) { + if (isReplaceableUser(use.getOwner())) + continue; + + OpBuilder builder(value.getContext()); + builder.setInsertionPoint(use.getOwner()); + + Location loc = value.getLoc(); + use.set(builder.create<tensor::CastOp>(loc, oldType, value)); + } + } + + oldTypes.clear(); + } + +private: + // A record of each value whose type was updated along with that value's + // previous type. + llvm::SmallVector<std::pair<Value, Type>> oldTypes; +}; + +void propagateShapesInRegion(Region ®ion, TypeModificationState &state); + +void propagateShapesToTosaIf(Operation &op, TypeModificationState &state) { IfOp ifOp = dyn_cast<IfOp>(op); if (!ifOp) return; @@ -58,7 +131,7 @@ void propagateShapesToTosaIf(Operation &op) { if (inferredTy.hasRank()) { Type newType = oldType.clone(inferredTy.getShape()); - blockArg.setType(newType); + state.setType(blockArg, newType); } } @@ -71,14 +144,14 @@ void propagateShapesToTosaIf(Operation &op) { ValueKnowledge::join(operandKnowledge, blockKnowledge); if (!joinedKnowledge) continue; - frontBlock.getArgument(i).setType(joinedKnowledge.getType()); + state.setType(frontBlock.getArgument(i), joinedKnowledge.getType()); } - propagateShapesInRegion(region); + propagateShapesInRegion(region, state); } } -void propagateShapesToTosaWhile(Operation &op) { +void propagateShapesToTosaWhile(Operation &op, TypeModificationState &state) { WhileOp whileOp = dyn_cast<WhileOp>(op); if (!whileOp) return; @@ -86,49 +159,29 @@ void propagateShapesToTosaWhile(Operation &op) { // Determine what the expected argument types are to the cond/body blocks. // The expected arguments should be compatible with ever iteration of the // loop body / condition for tosa.while. - llvm::SmallVector<Type> argTypes; - for (auto operand : op.getOperands()) { - auto operandTy = cast<ShapedType>(operand.getType()); - if (operandTy.hasRank()) { - auto newTy = operandTy.clone(operandTy.getShape()); - argTypes.push_back(newTy); - } else { - argTypes.push_back(operand.getType()); - } - } - - // Save out the type information so we can restore at the end. - llvm::DenseMap<Value, Type> originalTypeMap; - for (auto &block : op.getRegion(1)) { - for (auto arg : block.getArguments()) - originalTypeMap[arg] = arg.getType(); - for (auto &op : block) - for (auto result : op.getResults()) - originalTypeMap[result] = result.getType(); - } + SmallVector<Type> argTypes = llvm::to_vector(op.getOperandTypes()); bool hasNewTypes = true; while (hasNewTypes) { + TypeModificationState localState; // Set types on the block args. Region &bodyRegion = op.getRegion(1); Block &block = bodyRegion.front(); for (int i = 0, s = argTypes.size(); i < s; i++) { - block.getArgument(i).setType(argTypes[i]); + localState.setType(block.getArgument(i), argTypes[i]); } // Propagate to the end. - propagateShapesInRegion(bodyRegion); + propagateShapesInRegion(bodyRegion, localState); - // Find all the tosa yield types and verify there is atleast one. + // Find all the tosa yield types and verify there is a single one. llvm::SmallVector<YieldOp> yieldOps; for (auto &block : bodyRegion) if (auto yieldOp = dyn_cast<YieldOp>(block.getTerminator())) yieldOps.push_back(yieldOp); - if (yieldOps.empty()) - return; - + assert(yieldOps.size() == 1 && "missing or non-unique yield op"); // Using the new tosa.yield operand types, infer the new subtypes. llvm::SmallVector<ValueKnowledge> yieldTypeInfo; for (auto ty : argTypes) { @@ -158,17 +211,8 @@ void propagateShapesToTosaWhile(Operation &op) { argTypes[i] = newType; } - // The types inferred in the block assume the operand types specified for - // this iteration. We need to restore the original types to ensure that - // future iterations only use the already specified types, not possible - // types from previous iterations. - for (auto &block : bodyRegion) { - for (auto arg : block.getArguments()) - arg.setType(originalTypeMap[arg]); - for (auto &op : block) - for (auto result : op.getResults()) - result.setType(originalTypeMap[result]); - } + // Revert all changes made during the speculative part of the algorithm. + localState.revert(); } // We now set the block arguments according to the most recent shape @@ -176,41 +220,22 @@ void propagateShapesToTosaWhile(Operation &op) { // iteration. for (auto ®ion : op.getRegions()) { for (unsigned int i = 0, s = argTypes.size(); i < s; i++) { - region.front().getArgument(i).setType(argTypes[i]); + state.setType(region.front().getArgument(i), argTypes[i]); } - propagateShapesInRegion(region); + propagateShapesInRegion(region, state); } } -// Track the old type for each operand whose type was updated -// during inference. This information is used to introduce casts -// back to the type expected by the operand after inference. -struct TypeRewriteInfo { - OpOperand *operand; - Type oldType; -}; - -void propagateShapesInRegion(Region ®ion) { - // Check whether this use case is replaceable. We define an op as - // being replaceable if it is used by a TosaOp, or an op with a - // type-inference related interface. - // When a non-replaceable use is encountered, the value is wrapped in a - // cast back to the original type after inference. - auto isReplaceableUser = [](Operation *user) -> bool { - return user->getDialect()->getNamespace() == - TosaDialect::getDialectNamespace() || - isa<InferTypeOpInterface, InferShapedTypeOpInterface>(user); - }; - - llvm::SmallVector<TypeRewriteInfo> requiresUpdate; +void propagateShapesInRegion(Region ®ion, TypeModificationState &state) { for (auto &block : region) { for (Operation &op : block) { - if (op.getDialect()->getNamespace() != TosaDialect::getDialectNamespace()) + if (!op.getDialect() || + op.getDialect()->getNamespace() != TosaDialect::getDialectNamespace()) continue; - propagateShapesToTosaIf(op); - propagateShapesToTosaWhile(op); + propagateShapesToTosaIf(op, state); + propagateShapesToTosaWhile(op, state); InferShapedTypeOpInterface shapeInterface = dyn_cast<InferShapedTypeOpInterface>(op); @@ -252,30 +277,11 @@ void propagateShapesInRegion(Region ®ion) { continue; // Set new type - result.setType(newKnowledge.getType()); - - // Collect all uses of the operation which require update. - for (auto &user : result.getUses()) { - if (!isReplaceableUser(user.getOwner())) - requiresUpdate.push_back({&user, resultTy}); - } + state.setType(result, newKnowledge.getType()); } } } } - - // For each use whose type changed, cast the value with the new type back to - // the old type. - IRRewriter rewriter(region.getContext()); - for (auto [operand, oldType] : requiresUpdate) { - rewriter.setInsertionPoint(operand->getOwner()); - - auto oldValue = operand->get(); - - auto loc = oldValue.getLoc(); - auto castOp = rewriter.create<tensor::CastOp>(loc, oldType, oldValue); - operand->set(castOp); - } } /// Pass that performs shape propagation across TOSA operations. This includes @@ -285,7 +291,9 @@ struct TosaInferShapes public: void runOnOperation() override { func::FuncOp func = getOperation(); - propagateShapesInRegion(func.getBody()); + TypeModificationState state; + propagateShapesInRegion(func.getBody(), state); + state.commit(); } }; } // namespace diff --git a/mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp b/mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp index 6d7e3bc..52359fa8 100644 --- a/mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp +++ b/mlir/lib/Dialect/Vector/IR/ScalableValueBoundsConstraintSet.cpp @@ -47,17 +47,26 @@ ScalableValueBoundsConstraintSet::computeScalableBound( unsigned vscaleMax, presburger::BoundType boundType, bool closedUB, StopConditionFn stopCondition) { using namespace presburger; - assert(vscaleMin <= vscaleMax); - ScalableValueBoundsConstraintSet scalableCstr(value.getContext(), vscaleMin, - vscaleMax); - int64_t pos = scalableCstr.populateConstraintsSet(value, dim, stopCondition); + // No stop condition specified: Keep adding constraints until the worklist + // is empty. + auto defaultStopCondition = [&](Value v, std::optional<int64_t> dim, + mlir::ValueBoundsConstraintSet &cstr) { + return false; + }; + + ScalableValueBoundsConstraintSet scalableCstr( + value.getContext(), stopCondition ? stopCondition : defaultStopCondition, + vscaleMin, vscaleMax); + int64_t pos = scalableCstr.populateConstraintsSet(value, dim); // Project out all variables apart from vscale. // This should result in constraints in terms of vscale only. - scalableCstr.projectOut( - [&](ValueDim p) { return p.first != scalableCstr.getVscaleValue(); }); + auto projectOutFn = [&](ValueDim p) { + return p.first != scalableCstr.getVscaleValue(); + }; + scalableCstr.projectOut(projectOutFn); assert(scalableCstr.cstr.getNumDimAndSymbolVars() == scalableCstr.positionToValueDim.size() && diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp index 593c1e5..8d733c5 100644 --- a/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/VectorDropLeadUnitDim.cpp @@ -398,13 +398,30 @@ mlir::vector::castAwayContractionLeadingOneDim(vector::ContractionOp contractOp, transposeResults.push_back(targetExpr); } } + + // Checks if only the outer, unit dimensions (of size 1) are permuted. + // Such transposes do not materially effect the underlying vector and can + // be omitted. EG: perm [1, 0, 2] applied to vector<1x1x8xi32> + bool transposeNonOuterUnitDims = false; + auto operandShape = operands[it.index()].getType().cast<ShapedType>(); + for (auto [index, dim] : + llvm::enumerate(ArrayRef<int64_t>(perm).drop_back(1))) { + if (dim != static_cast<int64_t>(index) && + operandShape.getDimSize(index) != 1) { + transposeNonOuterUnitDims = true; + break; + } + } + // Do the tranpose now if needed so that we can drop the // correct dim using extract later. if (tranposeNeeded) { map = AffineMap::get(map.getNumDims(), 0, transposeResults, contractOp.getContext()); - operands[it.index()] = rewriter.create<vector::TransposeOp>( - loc, operands[it.index()], perm); + if (transposeNonOuterUnitDims) { + operands[it.index()] = rewriter.createOrFold<vector::TransposeOp>( + loc, operands[it.index()], perm); + } } } // We have taken care to have the dim to be dropped be diff --git a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp index 4fa5b8a..b59e906 100644 --- a/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp +++ b/mlir/lib/Dialect/Vector/Transforms/VectorLinearize.cpp @@ -26,6 +26,9 @@ static bool isLessThanTargetBitWidth(Operation *op, unsigned targetBitWidth) { // Reject index since getElementTypeBitWidth will abort for Index types. if (!vecType || vecType.getElementType().isIndex()) return false; + // There are no dimension to fold if it is a 0-D vector. + if (vecType.getRank() == 0) + return false; unsigned trailingVecDimBitWidth = vecType.getShape().back() * vecType.getElementTypeBitWidth(); if (trailingVecDimBitWidth >= targetBitWidth) diff --git a/mlir/lib/IR/PatternMatch.cpp b/mlir/lib/IR/PatternMatch.cpp index 5944a0e..286f47c 100644 --- a/mlir/lib/IR/PatternMatch.cpp +++ b/mlir/lib/IR/PatternMatch.cpp @@ -11,6 +11,7 @@ #include "mlir/IR/IRMapping.h" #include "mlir/IR/Iterators.h" #include "mlir/IR/RegionKindInterface.h" +#include "llvm/ADT/SmallPtrSet.h" using namespace mlir; @@ -250,6 +251,14 @@ void RewriterBase::finalizeOpModification(Operation *op) { rewriteListener->notifyOperationModified(op); } +void RewriterBase::replaceAllUsesExcept( + Value from, Value to, const SmallPtrSetImpl<Operation *> &preservedUsers) { + return replaceUsesWithIf(from, to, [&](OpOperand &use) { + Operation *user = use.getOwner(); + return !preservedUsers.contains(user); + }); +} + void RewriterBase::replaceUsesWithIf(Value from, Value to, function_ref<bool(OpOperand &)> functor, bool *allUsesReplaced) { diff --git a/mlir/lib/Interfaces/DataLayoutInterfaces.cpp b/mlir/lib/Interfaces/DataLayoutInterfaces.cpp index b5b7d78..e93a9ef 100644 --- a/mlir/lib/Interfaces/DataLayoutInterfaces.cpp +++ b/mlir/lib/Interfaces/DataLayoutInterfaces.cpp @@ -234,6 +234,15 @@ std::optional<uint64_t> mlir::detail::getDefaultIndexBitwidth( return std::nullopt; } +// Returns the endianness if specified in the given entry. If the entry is empty +// the default endianness represented by an empty attribute is returned. +Attribute mlir::detail::getDefaultEndianness(DataLayoutEntryInterface entry) { + if (entry == DataLayoutEntryInterface()) + return Attribute(); + + return entry.getValue(); +} + // Returns the memory space used for alloca operations if specified in the // given entry. If the entry is empty the default memory space represented by // an empty attribute is returned. @@ -548,6 +557,22 @@ std::optional<uint64_t> mlir::DataLayout::getTypeIndexBitwidth(Type t) const { }); } +mlir::Attribute mlir::DataLayout::getEndianness() const { + checkValid(); + if (endianness) + return *endianness; + DataLayoutEntryInterface entry; + if (originalLayout) + entry = originalLayout.getSpecForIdentifier( + originalLayout.getEndiannessIdentifier(originalLayout.getContext())); + + if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope)) + endianness = iface.getEndianness(entry); + else + endianness = detail::getDefaultEndianness(entry); + return *endianness; +} + mlir::Attribute mlir::DataLayout::getAllocaMemorySpace() const { checkValid(); if (allocaMemorySpace) diff --git a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp index 99598f2..0d362c7 100644 --- a/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp +++ b/mlir/lib/Interfaces/ValueBoundsOpInterface.cpp @@ -67,8 +67,11 @@ static std::optional<int64_t> getConstantIntValue(OpFoldResult ofr) { return std::nullopt; } -ValueBoundsConstraintSet::ValueBoundsConstraintSet(MLIRContext *ctx) - : builder(ctx) {} +ValueBoundsConstraintSet::ValueBoundsConstraintSet( + MLIRContext *ctx, StopConditionFn stopCondition) + : builder(ctx), stopCondition(stopCondition) { + assert(stopCondition && "expected non-null stop condition"); +} char ValueBoundsConstraintSet::ID = 0; @@ -193,7 +196,8 @@ static Operation *getOwnerOfValue(Value value) { return value.getDefiningOp(); } -void ValueBoundsConstraintSet::processWorklist(StopConditionFn stopCondition) { +void ValueBoundsConstraintSet::processWorklist() { + LLVM_DEBUG(llvm::dbgs() << "Processing value bounds worklist...\n"); while (!worklist.empty()) { int64_t pos = worklist.front(); worklist.pop(); @@ -214,13 +218,19 @@ void ValueBoundsConstraintSet::processWorklist(StopConditionFn stopCondition) { // Do not process any further if the stop condition is met. auto maybeDim = dim == kIndexValue ? std::nullopt : std::make_optional(dim); - if (stopCondition(value, maybeDim)) + if (stopCondition(value, maybeDim, *this)) { + LLVM_DEBUG(llvm::dbgs() << "Stop condition met for: " << value + << " (dim: " << maybeDim << ")\n"); continue; + } // Query `ValueBoundsOpInterface` for constraints. New items may be added to // the worklist. auto valueBoundsOp = dyn_cast<ValueBoundsOpInterface>(getOwnerOfValue(value)); + LLVM_DEBUG(llvm::dbgs() + << "Query value bounds for: " << value + << " (owner: " << getOwnerOfValue(value)->getName() << ")\n"); if (valueBoundsOp) { if (dim == kIndexValue) { valueBoundsOp.populateBoundsForIndexValue(value, *this); @@ -229,6 +239,7 @@ void ValueBoundsConstraintSet::processWorklist(StopConditionFn stopCondition) { } continue; } + LLVM_DEBUG(llvm::dbgs() << "--> ValueBoundsOpInterface not implemented\n"); // If the op does not implement `ValueBoundsOpInterface`, check if it // implements the `DestinationStyleOpInterface`. OpResults of such ops are @@ -278,32 +289,20 @@ LogicalResult ValueBoundsConstraintSet::computeBound( bool closedUB) { #ifndef NDEBUG assertValidValueDim(value, dim); - assert(!stopCondition(value, dim) && - "stop condition should not be satisfied for starting point"); #endif // NDEBUG int64_t ubAdjustment = closedUB ? 0 : 1; Builder b(value.getContext()); mapOperands.clear(); - if (stopCondition(value, dim)) { - // Special case: If the stop condition is satisfied for the input - // value/dimension, directly return it. - mapOperands.push_back(std::make_pair(value, dim)); - AffineExpr bound = b.getAffineDimExpr(0); - if (type == BoundType::UB) - bound = bound + ubAdjustment; - resultMap = AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, - b.getAffineDimExpr(0)); - return success(); - } - // Process the backward slice of `value` (i.e., reverse use-def chain) until // `stopCondition` is met. ValueDim valueDim = std::make_pair(value, dim.value_or(kIndexValue)); - ValueBoundsConstraintSet cstr(value.getContext()); + ValueBoundsConstraintSet cstr(value.getContext(), stopCondition); + assert(!stopCondition(value, dim, cstr) && + "stop condition should not be satisfied for starting point"); int64_t pos = cstr.insert(value, dim, /*isSymbol=*/false); - cstr.processWorklist(stopCondition); + cstr.processWorklist(); // Project out all variables (apart from `valueDim`) that do not match the // stop condition. @@ -313,7 +312,7 @@ LogicalResult ValueBoundsConstraintSet::computeBound( return false; auto maybeDim = p.second == kIndexValue ? std::nullopt : std::make_optional(p.second); - return !stopCondition(p.first, maybeDim); + return !stopCondition(p.first, maybeDim, cstr); }); // Compute lower and upper bounds for `valueDim`. @@ -419,7 +418,7 @@ LogicalResult ValueBoundsConstraintSet::computeDependentBound( bool closedUB) { return computeBound( resultMap, mapOperands, type, value, dim, - [&](Value v, std::optional<int64_t> d) { + [&](Value v, std::optional<int64_t> d, ValueBoundsConstraintSet &cstr) { return llvm::is_contained(dependencies, std::make_pair(v, d)); }, closedUB); @@ -455,7 +454,9 @@ LogicalResult ValueBoundsConstraintSet::computeIndependentBound( // Reify bounds in terms of any independent values. return computeBound( resultMap, mapOperands, type, value, dim, - [&](Value v, std::optional<int64_t> d) { return isIndependent(v); }, + [&](Value v, std::optional<int64_t> d, ValueBoundsConstraintSet &cstr) { + return isIndependent(v); + }, closedUB); } @@ -488,21 +489,19 @@ FailureOr<int64_t> ValueBoundsConstraintSet::computeConstantBound( presburger::BoundType type, AffineMap map, ValueDimList operands, StopConditionFn stopCondition, bool closedUB) { assert(map.getNumResults() == 1 && "expected affine map with one result"); - ValueBoundsConstraintSet cstr(map.getContext()); - int64_t pos = 0; - if (stopCondition) { - cstr.populateConstraintsSet(map, operands, stopCondition, &pos); - } else { - // No stop condition specified: Keep adding constraints until a bound could - // be computed. - cstr.populateConstraintsSet( - map, operands, - [&](Value v, std::optional<int64_t> dim) { - return cstr.cstr.getConstantBound64(type, pos).has_value(); - }, - &pos); - } + // Default stop condition if none was specified: Keep adding constraints until + // a bound could be computed. + int64_t pos; + auto defaultStopCondition = [&](Value v, std::optional<int64_t> dim, + ValueBoundsConstraintSet &cstr) { + return cstr.cstr.getConstantBound64(type, pos).has_value(); + }; + + ValueBoundsConstraintSet cstr( + map.getContext(), stopCondition ? stopCondition : defaultStopCondition); + cstr.populateConstraintsSet(map, operands, &pos); + // Compute constant bound for `valueDim`. int64_t ubAdjustment = closedUB ? 0 : 1; if (auto bound = cstr.cstr.getConstantBound64(type, pos)) @@ -510,8 +509,9 @@ FailureOr<int64_t> ValueBoundsConstraintSet::computeConstantBound( return failure(); } -int64_t ValueBoundsConstraintSet::populateConstraintsSet( - Value value, std::optional<int64_t> dim, StopConditionFn stopCondition) { +int64_t +ValueBoundsConstraintSet::populateConstraintsSet(Value value, + std::optional<int64_t> dim) { #ifndef NDEBUG assertValidValueDim(value, dim); #endif // NDEBUG @@ -519,12 +519,12 @@ int64_t ValueBoundsConstraintSet::populateConstraintsSet( AffineMap map = AffineMap::get(/*dimCount=*/1, /*symbolCount=*/0, Builder(value.getContext()).getAffineDimExpr(0)); - return populateConstraintsSet(map, {{value, dim}}, stopCondition); + return populateConstraintsSet(map, {{value, dim}}); } -int64_t ValueBoundsConstraintSet::populateConstraintsSet( - AffineMap map, ValueDimList operands, StopConditionFn stopCondition, - int64_t *posOut) { +int64_t ValueBoundsConstraintSet::populateConstraintsSet(AffineMap map, + ValueDimList operands, + int64_t *posOut) { assert(map.getNumResults() == 1 && "expected affine map with one result"); int64_t pos = insert(/*isSymbol=*/false); if (posOut) @@ -545,13 +545,7 @@ int64_t ValueBoundsConstraintSet::populateConstraintsSet( // Process the backward slice of `operands` (i.e., reverse use-def chain) // until `stopCondition` is met. - if (stopCondition) { - processWorklist(stopCondition); - } else { - // No stop condition specified: Keep adding constraints until the worklist - // is empty. - processWorklist([](Value v, std::optional<int64_t> dim) { return false; }); - } + processWorklist(); return pos; } diff --git a/mlir/lib/Support/Timing.cpp b/mlir/lib/Support/Timing.cpp index 2249312..ac16eb7 100644 --- a/mlir/lib/Support/Timing.cpp +++ b/mlir/lib/Support/Timing.cpp @@ -32,6 +32,7 @@ using namespace mlir; using namespace detail; using DisplayMode = DefaultTimingManager::DisplayMode; +using OutputFormat = DefaultTimingManager::OutputFormat; constexpr llvm::StringLiteral kTimingDescription = "... Execution time report ..."; @@ -109,56 +110,105 @@ TimingIdentifier TimingIdentifier::get(StringRef str, TimingManager &tm) { namespace { -/// Simple record class to record timing information. -struct TimeRecord { - TimeRecord(double wall = 0.0, double user = 0.0) : wall(wall), user(user) {} +class OutputTextStrategy : public OutputStrategy { +public: + OutputTextStrategy(raw_ostream &os) : OutputStrategy(os) {} + + void printHeader(const TimeRecord &total) override { + // Figure out how many spaces to description name. + unsigned padding = (80 - kTimingDescription.size()) / 2; + os << "===" << std::string(73, '-') << "===\n"; + os.indent(padding) << kTimingDescription << '\n'; + os << "===" << std::string(73, '-') << "===\n"; - TimeRecord &operator+=(const TimeRecord &other) { - wall += other.wall; - user += other.user; - return *this; + // Print the total time followed by the section headers. + os << llvm::format(" Total Execution Time: %.4f seconds\n\n", total.wall); + if (total.user != total.wall) + os << " ----User Time----"; + os << " ----Wall Time---- ----Name----\n"; } - TimeRecord &operator-=(const TimeRecord &other) { - wall -= other.wall; - user -= other.user; - return *this; + void printFooter() override { os.flush(); } + + void printTime(const TimeRecord &time, const TimeRecord &total) override { + if (total.user != total.wall) { + os << llvm::format(" %8.4f (%5.1f%%)", time.user, + 100.0 * time.user / total.user); + } + os << llvm::format(" %8.4f (%5.1f%%) ", time.wall, + 100.0 * time.wall / total.wall); } - /// Print the current time record to 'os', with a breakdown showing - /// contributions to the give 'total' time record. - void print(raw_ostream &os, const TimeRecord &total) { - if (total.user != total.wall) - os << llvm::format(" %8.4f (%5.1f%%)", user, 100.0 * user / total.user); - os << llvm::format(" %8.4f (%5.1f%%) ", wall, 100.0 * wall / total.wall); + void printListEntry(StringRef name, const TimeRecord &time, + const TimeRecord &total, bool lastEntry) override { + printTime(time, total); + os << name << "\n"; + } + + void printTreeEntry(unsigned indent, StringRef name, const TimeRecord &time, + const TimeRecord &total) override { + printTime(time, total); + os.indent(indent) << name << "\n"; } - double wall, user; + void printTreeEntryEnd(unsigned indent, bool lastEntry) override {} }; -} // namespace +class OutputJsonStrategy : public OutputStrategy { +public: + OutputJsonStrategy(raw_ostream &os) : OutputStrategy(os) {} -/// Utility to print a single line entry in the timer output. -static void printTimeEntry(raw_ostream &os, unsigned indent, StringRef name, - TimeRecord time, TimeRecord total) { - time.print(os, total); - os.indent(indent) << name << "\n"; -} + void printHeader(const TimeRecord &total) override { os << "[" << "\n"; } -/// Utility to print the timer heading information. -static void printTimeHeader(raw_ostream &os, TimeRecord total) { - // Figure out how many spaces to description name. - unsigned padding = (80 - kTimingDescription.size()) / 2; - os << "===" << std::string(73, '-') << "===\n"; - os.indent(padding) << kTimingDescription << '\n'; - os << "===" << std::string(73, '-') << "===\n"; - - // Print the total time followed by the section headers. - os << llvm::format(" Total Execution Time: %.4f seconds\n\n", total.wall); - if (total.user != total.wall) - os << " ----User Time----"; - os << " ----Wall Time---- ----Name----\n"; -} + void printFooter() override { + os << "]" << "\n"; + os.flush(); + } + + void printTime(const TimeRecord &time, const TimeRecord &total) override { + if (total.user != total.wall) { + os << "\"user\": {"; + os << "\"duration\": " << llvm::format("%8.4f", time.user) << ", "; + os << "\"percentage\": " + << llvm::format("%5.1f", 100.0 * time.user / total.user); + os << "}, "; + } + os << "\"wall\": {"; + os << "\"duration\": " << llvm::format("%8.4f", time.wall) << ", "; + os << "\"percentage\": " + << llvm::format("%5.1f", 100.0 * time.wall / total.wall); + os << "}"; + } + + void printListEntry(StringRef name, const TimeRecord &time, + const TimeRecord &total, bool lastEntry) override { + os << "{"; + printTime(time, total); + os << ", \"name\": " << "\"" << name << "\""; + os << "}"; + if (!lastEntry) + os << ","; + os << "\n"; + } + + void printTreeEntry(unsigned indent, StringRef name, const TimeRecord &time, + const TimeRecord &total) override { + os.indent(indent) << "{"; + printTime(time, total); + os << ", \"name\": " << "\"" << name << "\""; + os << ", \"passes\": [" << "\n"; + } + + void printTreeEntryEnd(unsigned indent, bool lastEntry) override { + os.indent(indent) << "{}]"; + os << "}"; + if (!lastEntry) + os << ","; + os << "\n"; + } +}; + +} // namespace //===----------------------------------------------------------------------===// // Timer Implementation for DefaultTimingManager @@ -176,7 +226,8 @@ public: using ChildrenMap = llvm::MapVector<const void *, std::unique_ptr<TimerImpl>>; using AsyncChildrenMap = llvm::DenseMap<uint64_t, ChildrenMap>; - TimerImpl(std::string &&name) : threadId(llvm::get_threadid()), name(name) {} + TimerImpl(std::string &&name, std::unique_ptr<OutputStrategy> &output) + : threadId(llvm::get_threadid()), name(name), output(output) {} /// Start the timer. void start() { startTime = std::chrono::steady_clock::now(); } @@ -206,7 +257,7 @@ public: TimerImpl *nestTail(std::unique_ptr<TimerImpl> &child, function_ref<std::string()> nameBuilder) { if (!child) - child = std::make_unique<TimerImpl>(nameBuilder()); + child = std::make_unique<TimerImpl>(nameBuilder(), output); return child.get(); } @@ -320,7 +371,7 @@ public: } /// Print the timing result in list mode. - void printAsList(raw_ostream &os, TimeRecord total) { + void printAsList(TimeRecord total) { // Flatten the leaf timers in the tree and merge them by name. llvm::StringMap<TimeRecord> mergedTimers; std::function<void(TimerImpl *)> addTimer = [&](TimerImpl *timer) { @@ -343,34 +394,37 @@ public: // Print the timing information sequentially. for (auto &timeData : timerNameAndTime) - printTimeEntry(os, 0, timeData.first, timeData.second, total); + output->printListEntry(timeData.first, timeData.second, total); } /// Print the timing result in tree mode. - void printAsTree(raw_ostream &os, TimeRecord total, unsigned indent = 0) { + void printAsTree(TimeRecord total, unsigned indent = 0) { unsigned childIndent = indent; if (!hidden) { - printTimeEntry(os, indent, name, getTimeRecord(), total); + output->printTreeEntry(indent, name, getTimeRecord(), total); childIndent += 2; } for (auto &child : children) { - child.second->printAsTree(os, total, childIndent); + child.second->printAsTree(total, childIndent); + } + if (!hidden) { + output->printTreeEntryEnd(indent); } } /// Print the current timing information. - void print(raw_ostream &os, DisplayMode displayMode) { + void print(DisplayMode displayMode) { // Print the banner. auto total = getTimeRecord(); - printTimeHeader(os, total); + output->printHeader(total); // Defer to a specialized printer for each display mode. switch (displayMode) { case DisplayMode::List: - printAsList(os, total); + printAsList(total); break; case DisplayMode::Tree: - printAsTree(os, total); + printAsTree(total); break; } @@ -379,9 +433,9 @@ public: auto rest = total; for (auto &child : children) rest -= child.second->getTimeRecord(); - printTimeEntry(os, 0, "Rest", rest, total); - printTimeEntry(os, 0, "Total", total, total); - os.flush(); + output->printListEntry("Rest", rest, total); + output->printListEntry("Total", total, total, /*lastEntry=*/true); + output->printFooter(); } /// The last time instant at which the timer was started. @@ -415,6 +469,8 @@ public: /// Mutex for the async children. std::mutex asyncMutex; + + std::unique_ptr<OutputStrategy> &output; }; } // namespace @@ -435,9 +491,6 @@ public: /// The configured display mode. DisplayMode displayMode = DisplayMode::Tree; - /// The stream where we should print our output. This will always be non-null. - raw_ostream *output = &llvm::errs(); - /// The root timer. std::unique_ptr<TimerImpl> rootTimer; }; @@ -446,7 +499,8 @@ public: } // namespace mlir DefaultTimingManager::DefaultTimingManager() - : impl(std::make_unique<DefaultTimingManagerImpl>()) { + : impl(std::make_unique<DefaultTimingManagerImpl>()), + out(std::make_unique<OutputTextStrategy>(llvm::errs())) { clear(); // initializes the root timer } @@ -469,26 +523,22 @@ DefaultTimingManager::DisplayMode DefaultTimingManager::getDisplayMode() const { } /// Change the stream where the output will be printed to. -void DefaultTimingManager::setOutput(raw_ostream &os) { impl->output = &os; } - -/// Return the current output stream where the output will be printed to. -raw_ostream &DefaultTimingManager::getOutput() const { - assert(impl->output); - return *impl->output; +void DefaultTimingManager::setOutput(std::unique_ptr<OutputStrategy> output) { + out = std::move(output); } /// Print and clear the timing results. void DefaultTimingManager::print() { if (impl->enabled) { impl->rootTimer->finalize(); - impl->rootTimer->print(*impl->output, impl->displayMode); + impl->rootTimer->print(impl->displayMode); } clear(); } /// Clear the timing results. void DefaultTimingManager::clear() { - impl->rootTimer = std::make_unique<TimerImpl>("root"); + impl->rootTimer = std::make_unique<TimerImpl>("root", out); impl->rootTimer->hidden = true; } @@ -500,13 +550,13 @@ void DefaultTimingManager::dumpTimers(raw_ostream &os) { /// Debug print the timers as a list. void DefaultTimingManager::dumpAsList(raw_ostream &os) { impl->rootTimer->finalize(); - impl->rootTimer->print(os, DisplayMode::List); + impl->rootTimer->print(DisplayMode::List); } /// Debug print the timers as a tree. void DefaultTimingManager::dumpAsTree(raw_ostream &os) { impl->rootTimer->finalize(); - impl->rootTimer->print(os, DisplayMode::Tree); + impl->rootTimer->print(DisplayMode::Tree); } std::optional<void *> DefaultTimingManager::rootTimer() { @@ -549,6 +599,13 @@ struct DefaultTimingManagerOptions { "display the results in a list sorted by total time"), clEnumValN(DisplayMode::Tree, "tree", "display the results ina with a nested tree view"))}; + llvm::cl::opt<OutputFormat> outputFormat{ + "mlir-output-format", llvm::cl::desc("Output format for timing data"), + llvm::cl::init(OutputFormat::Text), + llvm::cl::values(clEnumValN(OutputFormat::Text, "text", + "display the results in text format"), + clEnumValN(OutputFormat::Json, "json", + "display the results in JSON format"))}; }; } // namespace @@ -564,4 +621,11 @@ void mlir::applyDefaultTimingManagerCLOptions(DefaultTimingManager &tm) { return; tm.setEnabled(options->timing); tm.setDisplayMode(options->displayMode); + + std::unique_ptr<OutputStrategy> printer; + if (options->outputFormat == OutputFormat::Text) + printer = std::make_unique<OutputTextStrategy>(llvm::errs()); + else if (options->outputFormat == OutputFormat::Json) + printer = std::make_unique<OutputJsonStrategy>(llvm::errs()); + tm.setOutput(std::move(printer)); } diff --git a/mlir/lib/Target/Cpp/TranslateToCpp.cpp b/mlir/lib/Target/Cpp/TranslateToCpp.cpp index 0b07b4b..ee87c1d 100644 --- a/mlir/lib/Target/Cpp/TranslateToCpp.cpp +++ b/mlir/lib/Target/Cpp/TranslateToCpp.cpp @@ -1104,7 +1104,7 @@ CppEmitter::CppEmitter(raw_ostream &os, bool declareVariablesAtTop) std::string CppEmitter::getSubscriptName(emitc::SubscriptOp op) { std::string out; llvm::raw_string_ostream ss(out); - ss << getOrCreateName(op.getArray()); + ss << getOrCreateName(op.getValue()); for (auto index : op.getIndices()) { ss << "[" << getOrCreateName(index) << "]"; } diff --git a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp index 646d0ed..08ec578 100644 --- a/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp +++ b/mlir/lib/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.cpp @@ -804,13 +804,13 @@ convertOmpTaskgroupOp(omp::TaskgroupOp tgOp, llvm::IRBuilderBase &builder, /// Allocate space for privatized reduction variables. template <typename T> -static void -allocByValReductionVars(T loop, llvm::IRBuilderBase &builder, - LLVM::ModuleTranslation &moduleTranslation, - llvm::OpenMPIRBuilder::InsertPointTy &allocaIP, - SmallVector<omp::DeclareReductionOp> &reductionDecls, - SmallVector<llvm::Value *> &privateReductionVariables, - DenseMap<Value, llvm::Value *> &reductionVariableMap) { +static void allocByValReductionVars( + T loop, llvm::IRBuilderBase &builder, + LLVM::ModuleTranslation &moduleTranslation, + llvm::OpenMPIRBuilder::InsertPointTy &allocaIP, + SmallVectorImpl<omp::DeclareReductionOp> &reductionDecls, + SmallVectorImpl<llvm::Value *> &privateReductionVariables, + DenseMap<Value, llvm::Value *> &reductionVariableMap) { llvm::IRBuilderBase::InsertPointGuard guard(builder); builder.restoreIP(allocaIP); auto args = @@ -825,6 +825,25 @@ allocByValReductionVars(T loop, llvm::IRBuilderBase &builder, } } +/// Map input argument to all reduction initialization regions +template <typename T> +static void +mapInitializationArg(T loop, LLVM::ModuleTranslation &moduleTranslation, + SmallVectorImpl<omp::DeclareReductionOp> &reductionDecls, + unsigned i) { + // map input argument to the initialization region + mlir::omp::DeclareReductionOp &reduction = reductionDecls[i]; + Region &initializerRegion = reduction.getInitializerRegion(); + Block &entry = initializerRegion.front(); + assert(entry.getNumArguments() == 1 && + "the initialization region has one argument"); + + mlir::Value mlirSource = loop.getReductionVars()[i]; + llvm::Value *llvmSource = moduleTranslation.lookupValue(mlirSource); + assert(llvmSource && "lookup reduction var"); + moduleTranslation.mapValue(entry.getArgument(0), llvmSource); +} + /// Collect reduction info template <typename T> static void collectReductionInfo( @@ -858,6 +877,32 @@ static void collectReductionInfo( } } +/// handling of DeclareReductionOp's cleanup region +static LogicalResult inlineReductionCleanup( + llvm::SmallVectorImpl<omp::DeclareReductionOp> &reductionDecls, + llvm::ArrayRef<llvm::Value *> privateReductionVariables, + LLVM::ModuleTranslation &moduleTranslation, llvm::IRBuilderBase &builder) { + for (auto [i, reductionDecl] : llvm::enumerate(reductionDecls)) { + Region &cleanupRegion = reductionDecl.getCleanupRegion(); + if (cleanupRegion.empty()) + continue; + + // map the argument to the cleanup region + Block &entry = cleanupRegion.front(); + moduleTranslation.mapValue(entry.getArgument(0), + privateReductionVariables[i]); + + if (failed(inlineConvertOmpRegions(cleanupRegion, "omp.reduction.cleanup", + builder, moduleTranslation))) + return failure(); + + // clear block argument mapping in case it needs to be re-created with a + // different source for another use of the same reduction decl + moduleTranslation.forgetMapping(cleanupRegion); + } + return success(); +} + /// Converts an OpenMP workshare loop into LLVM IR using OpenMPIRBuilder. static LogicalResult convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder, @@ -902,6 +947,10 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder, loop.getRegion().getArguments().take_back(loop.getNumReductionVars()); for (unsigned i = 0; i < loop.getNumReductionVars(); ++i) { SmallVector<llvm::Value *> phis; + + // map block argument to initializer region + mapInitializationArg(loop, moduleTranslation, reductionDecls, i); + if (failed(inlineConvertOmpRegions(reductionDecls[i].getInitializerRegion(), "omp.reduction.neutral", builder, moduleTranslation, &phis))) @@ -925,6 +974,11 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder, builder.CreateStore(phis[0], privateReductionVariables[i]); // the rest was handled in allocByValReductionVars } + + // forget the mapping for the initializer region because we might need a + // different mapping if this reduction declaration is re-used for a + // different variable + moduleTranslation.forgetMapping(reductionDecls[i].getInitializerRegion()); } // Store the mapping between reduction variables and their private copies on @@ -1044,7 +1098,9 @@ convertOmpWsloop(Operation &opInst, llvm::IRBuilderBase &builder, tempTerminator->eraseFromParent(); builder.restoreIP(nextInsertionPoint); - return success(); + // after the workshare loop, deallocate private reduction variables + return inlineReductionCleanup(reductionDecls, privateReductionVariables, + moduleTranslation, builder); } /// A RAII class that on construction replaces the region arguments of the @@ -1097,13 +1153,13 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, LogicalResult bodyGenStatus = success(); llvm::OpenMPIRBuilder *ompBuilder = moduleTranslation.getOpenMPBuilder(); - auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) { - // Collect reduction declarations - SmallVector<omp::DeclareReductionOp> reductionDecls; - collectReductionDecls(opInst, reductionDecls); + // Collect reduction declarations + SmallVector<omp::DeclareReductionOp> reductionDecls; + collectReductionDecls(opInst, reductionDecls); + SmallVector<llvm::Value *> privateReductionVariables; + auto bodyGenCB = [&](InsertPointTy allocaIP, InsertPointTy codeGenIP) { // Allocate reduction vars - SmallVector<llvm::Value *> privateReductionVariables; DenseMap<Value, llvm::Value *> reductionVariableMap; if (!isByRef) { allocByValReductionVars(opInst, builder, moduleTranslation, allocaIP, @@ -1118,6 +1174,9 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, opInst.getNumReductionVars()); for (unsigned i = 0; i < opInst.getNumReductionVars(); ++i) { SmallVector<llvm::Value *> phis; + + // map the block argument + mapInitializationArg(opInst, moduleTranslation, reductionDecls, i); if (failed(inlineConvertOmpRegions( reductionDecls[i].getInitializerRegion(), "omp.reduction.neutral", builder, moduleTranslation, &phis))) @@ -1144,6 +1203,10 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, builder.CreateStore(phis[0], privateReductionVariables[i]); // the rest is done in allocByValReductionVars } + + // clear block argument mapping in case it needs to be re-created with a + // different source for another use of the same reduction decl + moduleTranslation.forgetMapping(reductionDecls[i].getInitializerRegion()); } // Store the mapping between reduction variables and their private copies on @@ -1296,7 +1359,18 @@ convertOmpParallel(omp::ParallelOp opInst, llvm::IRBuilderBase &builder, // TODO: Perform finalization actions for variables. This has to be // called for variables which have destructors/finalizers. - auto finiCB = [&](InsertPointTy codeGenIP) {}; + auto finiCB = [&](InsertPointTy codeGenIP) { + InsertPointTy oldIP = builder.saveIP(); + builder.restoreIP(codeGenIP); + + // if the reduction has a cleanup region, inline it here to finalize the + // reduction variables + if (failed(inlineReductionCleanup(reductionDecls, privateReductionVariables, + moduleTranslation, builder))) + bodyGenStatus = failure(); + + builder.restoreIP(oldIP); + }; llvm::Value *ifCond = nullptr; if (auto ifExprVar = opInst.getIfExprVar()) diff --git a/mlir/lib/Transforms/Mem2Reg.cpp b/mlir/lib/Transforms/Mem2Reg.cpp index 80e3b79..abe565e 100644 --- a/mlir/lib/Transforms/Mem2Reg.cpp +++ b/mlir/lib/Transforms/Mem2Reg.cpp @@ -202,6 +202,7 @@ private: /// Contains the reaching definition at this operation. Reaching definitions /// are only computed for promotable memory operations with blocking uses. DenseMap<PromotableMemOpInterface, Value> reachingDefs; + DenseMap<PromotableMemOpInterface, Value> replacedValuesMap; DominanceInfo &dominance; MemorySlotPromotionInfo info; const Mem2RegStatistics &statistics; @@ -438,6 +439,7 @@ Value MemorySlotPromoter::computeReachingDefInBlock(Block *block, assert(stored && "a memory operation storing to a slot must provide a " "new definition of the slot"); reachingDef = stored; + replacedValuesMap[memOp] = stored; } } } @@ -552,6 +554,10 @@ void MemorySlotPromoter::removeBlockingUses() { dominanceSort(usersToRemoveUses, *slot.ptr.getParentBlock()->getParent()); llvm::SmallVector<Operation *> toErase; + // List of all replaced values in the slot. + llvm::SmallVector<std::pair<Operation *, Value>> replacedValuesList; + // Ops to visit with the `visitReplacedValues` method. + llvm::SmallVector<PromotableOpInterface> toVisit; for (Operation *toPromote : llvm::reverse(usersToRemoveUses)) { if (auto toPromoteMemOp = dyn_cast<PromotableMemOpInterface>(toPromote)) { Value reachingDef = reachingDefs.lookup(toPromoteMemOp); @@ -565,7 +571,9 @@ void MemorySlotPromoter::removeBlockingUses() { slot, info.userToBlockingUses[toPromote], rewriter, reachingDef) == DeletionKind::Delete) toErase.push_back(toPromote); - + if (toPromoteMemOp.storesTo(slot)) + if (Value replacedValue = replacedValuesMap[toPromoteMemOp]) + replacedValuesList.push_back({toPromoteMemOp, replacedValue}); continue; } @@ -574,6 +582,12 @@ void MemorySlotPromoter::removeBlockingUses() { if (toPromoteBasic.removeBlockingUses(info.userToBlockingUses[toPromote], rewriter) == DeletionKind::Delete) toErase.push_back(toPromote); + if (toPromoteBasic.requiresReplacedValues()) + toVisit.push_back(toPromoteBasic); + } + for (PromotableOpInterface op : toVisit) { + rewriter.setInsertionPointAfter(op); + op.visitReplacedValues(replacedValuesList, rewriter); } for (Operation *toEraseOp : toErase) diff --git a/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir index 9793b2d..7aa2ba8 100644 --- a/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir +++ b/mlir/test/Conversion/MemRefToEmitC/memref-to-emitc.mlir @@ -6,7 +6,7 @@ func.func @memref_store(%v : f32, %i: index, %j: index) { // CHECK: %[[ALLOCA:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> !emitc.array<4x8xf32> %0 = memref.alloca() : memref<4x8xf32> - // CHECK: %[[SUBSCRIPT:.*]] = emitc.subscript %[[ALLOCA]][%[[i]], %[[j]]] : <4x8xf32> + // CHECK: %[[SUBSCRIPT:.*]] = emitc.subscript %[[ALLOCA]][%[[i]], %[[j]]] : (!emitc.array<4x8xf32>, index, index) -> f32 // CHECK: emitc.assign %[[v]] : f32 to %[[SUBSCRIPT:.*]] : f32 memref.store %v, %0[%i, %j] : memref<4x8xf32> return @@ -19,7 +19,7 @@ func.func @memref_load(%i: index, %j: index) -> f32 { // CHECK: %[[ALLOCA:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> !emitc.array<4x8xf32> %0 = memref.alloca() : memref<4x8xf32> - // CHECK: %[[LOAD:.*]] = emitc.subscript %[[ALLOCA]][%[[i]], %[[j]]] : <4x8xf32> + // CHECK: %[[LOAD:.*]] = emitc.subscript %[[ALLOCA]][%[[i]], %[[j]]] : (!emitc.array<4x8xf32>, index, index) -> f32 // CHECK: %[[VAR:.*]] = "emitc.variable"() <{value = #emitc.opaque<"">}> : () -> f32 // CHECK: emitc.assign %[[LOAD]] : f32 to %[[VAR]] : f32 %1 = memref.load %0[%i, %j] : memref<4x8xf32> diff --git a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir index 17eec59..ad65410 100644 --- a/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir +++ b/mlir/test/Conversion/TosaToLinalg/tosa-to-linalg-invalid.mlir @@ -15,3 +15,15 @@ func.func @tensor_with_unknown_rank(%arg0: tensor<*xi8>) -> tensor<*xi8> { %0 = "tosa.abs"(%arg0) : (tensor<*xi8>) -> tensor<*xi8> return %0 : tensor<*xi8> } + +// ----- + +// CHECK-LABEL: @unranked_add +func.func @unranked_add(%arg0 : tensor<10x10xf32> , %arg1 : tensor<10x10xf32>, %arg2 : tensor<*xf32>) -> (tensor<10x10xf32>) { + // expected-error@+3 {{failed to legalize operation 'tosa.add'}} + %reduce = tosa.reduce_max %arg0 {axis = 1 : i32} : (tensor<10x10xf32>) -> tensor<10x1xf32> + %1 = tosa.add %reduce, %arg1 : (tensor<10x1xf32>, tensor<10x10xf32>) -> tensor<10x10xf32> + %0 = tosa.add %1, %arg2 : (tensor<10x10xf32>, tensor<*xf32>) -> tensor<*xf32> + %2 = tosa.reshape %0 {new_shape = array<i64: 10, 10>} : (tensor<*xf32>) -> tensor<10x10xf32> + return %2 : tensor<10x10xf32> +} diff --git a/mlir/test/Dialect/Affine/SuperVectorize/compose_maps.mlir b/mlir/test/Dialect/Affine/SuperVectorize/compose_maps.mlir index b53fc55..d998ed8 100644 --- a/mlir/test/Dialect/Affine/SuperVectorize/compose_maps.mlir +++ b/mlir/test/Dialect/Affine/SuperVectorize/compose_maps.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt -allow-unregistered-dialect %s -affine-super-vectorizer-test -compose-maps -split-input-file 2>&1 | FileCheck %s +// RUN: mlir-opt -allow-unregistered-dialect %s -affine-super-vectorizer-test=compose-maps -split-input-file 2>&1 | FileCheck %s // For all these cases, the test traverses the `test_affine_map` ops and // composes them in order one-by-one. diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vector_utils.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vector_utils.mlir index e58de9f..bd71164 100644 --- a/mlir/test/Dialect/Affine/SuperVectorize/vector_utils.mlir +++ b/mlir/test/Dialect/Affine/SuperVectorize/vector_utils.mlir @@ -1,6 +1,6 @@ -// RUN: mlir-opt %s -affine-super-vectorizer-test -vector-shape-ratio 4 -vector-shape-ratio 8 2>&1 | FileCheck %s -// RUN: mlir-opt %s -affine-super-vectorizer-test -vector-shape-ratio 2 -vector-shape-ratio 5 -vector-shape-ratio 2 2>&1 | FileCheck %s -check-prefix=TEST-3x4x5x8 -// RUN: mlir-opt %s -affine-super-vectorizer-test -vectorize-affine-loop-nest 2>&1 | FileCheck %s -check-prefix=VECNEST +// RUN: mlir-opt %s -affine-super-vectorizer-test="vector-shape-ratio=4,8" 2>&1 | FileCheck %s +// RUN: mlir-opt %s -affine-super-vectorizer-test="vector-shape-ratio=2,5,2" 2>&1 | FileCheck %s -check-prefix=TEST-3x4x5x8 +// RUN: mlir-opt %s -affine-super-vectorizer-test=vectorize-affine-loop-nest 2>&1 | FileCheck %s -check-prefix=VECNEST func.func @vector_add_2d(%arg0: index, %arg1: index) -> f32 { // Nothing should be matched in this first block. diff --git a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_unsupported.mlir b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_unsupported.mlir index c117bfc..6c1a7c4 100644 --- a/mlir/test/Dialect/Affine/SuperVectorize/vectorize_unsupported.mlir +++ b/mlir/test/Dialect/Affine/SuperVectorize/vectorize_unsupported.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -affine-super-vectorizer-test -vectorize-affine-loop-nest -split-input-file 2>&1 | FileCheck %s +// RUN: mlir-opt %s -affine-super-vectorizer-test=vectorize-affine-loop-nest -split-input-file 2>&1 | FileCheck %s func.func @unparallel_loop_reduction_unsupported(%in: memref<256x512xf32>, %out: memref<256xf32>) { // CHECK: Outermost loop cannot be parallel diff --git a/mlir/test/Dialect/Affine/loop-coalescing.mlir b/mlir/test/Dialect/Affine/loop-coalescing.mlir index 9c17fb2..ae0adf5 100644 --- a/mlir/test/Dialect/Affine/loop-coalescing.mlir +++ b/mlir/test/Dialect/Affine/loop-coalescing.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt -split-input-file -allow-unregistered-dialect -affine-loop-coalescing %s | FileCheck %s +// RUN: mlir-opt -split-input-file -allow-unregistered-dialect -affine-loop-coalescing --cse %s | FileCheck %s // CHECK-LABEL: @one_3d_nest func.func @one_3d_nest() { @@ -239,19 +239,15 @@ func.func @coalesce_affine_for(%arg0: memref<?x?xf32>) { } return } -// CHECK: %[[T0:.*]] = memref.dim %arg{{.*}}, %c{{.*}} : memref<?x?xf32> -// CHECK: %[[T1:.*]] = memref.dim %arg{{.*}}, %c{{.*}} : memref<?x?xf32> -// CHECK: %[[T2:.*]] = memref.dim %arg{{.*}}, %c{{.*}} : memref<?x?xf32> -// CHECK-DAG: %[[T3:.*]] = affine.apply #[[IDENTITY]]()[%[[T0]]] -// CHECK-DAG: %[[T4:.*]] = affine.apply #[[IDENTITY]]()[%[[T1]]] -// CHECK-DAG: %[[T5:.*]] = affine.apply #[[PRODUCT]](%[[T3]])[%[[T4]]] -// CHECK-DAG: %[[T6:.*]] = affine.apply #[[IDENTITY]]()[%[[T2]]] -// CHECK-DAG: %[[T7:.*]] = affine.apply #[[PRODUCT]](%[[T5]])[%[[T6]]] -// CHECK: affine.for %[[IV:.*]] = 0 to %[[T7]] -// CHECK-DAG: %[[K:.*]] = affine.apply #[[MOD]](%[[IV]])[%[[T6]]] -// CHECK-DAG: %[[T9:.*]] = affine.apply #[[FLOOR]](%[[IV]])[%[[T6]]] -// CHECK-DAG: %[[J:.*]] = affine.apply #[[MOD]](%[[T9]])[%[[T4]]] -// CHECK-DAG: %[[I:.*]] = affine.apply #[[FLOOR]](%[[T9]])[%[[T4]]] +// CHECK: %[[DIM:.*]] = memref.dim %arg{{.*}}, %c{{.*}} : memref<?x?xf32> +// CHECK-DAG: %[[T0:.*]] = affine.apply #[[IDENTITY]]()[%[[DIM]]] +// CHECK-DAG: %[[T1:.*]] = affine.apply #[[PRODUCT]](%[[T0]])[%[[T0]]] +// CHECK-DAG: %[[T2:.*]] = affine.apply #[[PRODUCT]](%[[T1]])[%[[T0]]] +// CHECK: affine.for %[[IV:.*]] = 0 to %[[T2]] +// CHECK-DAG: %[[K:.*]] = affine.apply #[[MOD]](%[[IV]])[%[[T0]]] +// CHECK-DAG: %[[T9:.*]] = affine.apply #[[FLOOR]](%[[IV]])[%[[T0]]] +// CHECK-DAG: %[[J:.*]] = affine.apply #[[MOD]](%[[T9]])[%[[T0]]] +// CHECK-DAG: %[[I:.*]] = affine.apply #[[FLOOR]](%[[T9]])[%[[T0]]] // CHECK-NEXT: "test.foo"(%[[I]], %[[J]], %[[K]]) // CHECK-NEXT: } // CHECK-NEXT: return @@ -277,18 +273,16 @@ func.func @coalesce_affine_for(%arg0: memref<?x?xf32>) { } return } -// CHECK: %[[T0:.*]] = memref.dim %arg{{.*}}, %c{{.*}} : memref<?x?xf32> -// CHECK: %[[T1:.*]] = memref.dim %arg{{.*}}, %c{{.*}} : memref<?x?xf32> -// CHECK-DAG: %[[T2:.*]] = affine.apply #[[IDENTITY]]()[%[[T0]]] -// CHECK-DAG: %[[T3:.*]] = affine.apply #[[IDENTITY]]()[%[[T1]]] -// CHECK-DAG: %[[T4:.*]] = affine.apply #[[PRODUCT]](%[[T2]])[%[[T3]]] -// CHECK-DAG: %[[T5:.*]] = affine.apply #[[SIXTY_FOUR]]() -// CHECK-DAG: %[[T6:.*]] = affine.apply #[[PRODUCT]](%[[T4]])[%[[T5]]] -// CHECK: affine.for %[[IV:.*]] = 0 to %[[T6]] -// CHECK-DAG: %[[K:.*]] = affine.apply #[[MOD]](%[[IV]])[%[[T5]]] -// CHECK-DAG: %[[T8:.*]] = affine.apply #[[DIV]](%[[IV]])[%[[T5]]] -// CHECK-DAG: %[[J:.*]] = affine.apply #[[MOD]](%[[T8]])[%[[T3]]] -// CHECK-DAG: %[[I:.*]] = affine.apply #[[DIV]](%[[T8]])[%[[T3]]] +// CHECK: %[[DIM:.*]] = memref.dim %arg{{.*}}, %c{{.*}} : memref<?x?xf32> +// CHECK-DAG: %[[T0:.*]] = affine.apply #[[IDENTITY]]()[%[[DIM]]] +// CHECK-DAG: %[[T1:.*]] = affine.apply #[[PRODUCT]](%[[T0]])[%[[T0]]] +// CHECK-DAG: %[[T2:.*]] = affine.apply #[[SIXTY_FOUR]]() +// CHECK-DAG: %[[T3:.*]] = affine.apply #[[PRODUCT]](%[[T1]])[%[[T2]]] +// CHECK: affine.for %[[IV:.*]] = 0 to %[[T3]] +// CHECK-DAG: %[[K:.*]] = affine.apply #[[MOD]](%[[IV]])[%[[T2]]] +// CHECK-DAG: %[[T5:.*]] = affine.apply #[[DIV]](%[[IV]])[%[[T2]]] +// CHECK-DAG: %[[J:.*]] = affine.apply #[[MOD]](%[[T5]])[%[[T0]]] +// CHECK-DAG: %[[I:.*]] = affine.apply #[[DIV]](%[[T5]])[%[[T0]]] // CHECK-NEXT: "test.foo"(%[[I]], %[[J]], %[[K]]) // CHECK-NEXT: } // CHECK-NEXT: return @@ -316,19 +310,16 @@ func.func @coalesce_affine_for(%arg0: memref<?x?xf32>) { } return } -// CHECK: %[[T0:.*]] = memref.dim %arg{{.*}}, %c{{.*}} : memref<?x?xf32> -// CHECK: %[[T1:.*]] = memref.dim %arg{{.*}}, %c{{.*}} : memref<?x?xf32> -// CHECK: %[[T2:.*]] = memref.dim %arg{{.*}}, %c{{.*}} : memref<?x?xf32> -// CHECK-DAG: %[[T3:.*]] = affine.min #[[MAP0]]()[%[[T0]]] -// CHECK-DAG: %[[T4:.*]] = affine.apply #[[IDENTITY]]()[%[[T1]]] -// CHECK-DAG: %[[T5:.*]] = affine.apply #[[PRODUCT]](%[[T3]])[%[[T4]]] -// CHECK-DAG: %[[T6:.*]] = affine.apply #[[IDENTITY]]()[%[[T2]]] -// CHECK-DAG: %[[T7:.*]] = affine.apply #[[PRODUCT]](%[[T5]])[%[[T6]]] -// CHECK: affine.for %[[IV:.*]] = 0 to %[[T7]] -// CHECK-DAG: %[[K:.*]] = affine.apply #[[MOD]](%[[IV]])[%[[T6]]] -// CHECK-DAG: %[[T9:.*]] = affine.apply #[[DIV]](%[[IV]])[%[[T6]]] -// CHECK-DAG: %[[J:.*]] = affine.apply #[[MOD]](%[[T9]])[%[[T4]]] -// CHECK-DAG: %[[I:.*]] = affine.apply #[[DIV]](%[[T9]])[%[[T4]]] +// CHECK: %[[DIM:.*]] = memref.dim %arg{{.*}}, %c{{.*}} : memref<?x?xf32> +// CHECK-DAG: %[[T0:.*]] = affine.min #[[MAP0]]()[%[[DIM]]] +// CHECK-DAG: %[[T1:.*]] = affine.apply #[[IDENTITY]]()[%[[DIM]]] +// CHECK-DAG: %[[T2:.*]] = affine.apply #[[PRODUCT]](%[[T0]])[%[[T1]]] +// CHECK-DAG: %[[T3:.*]] = affine.apply #[[PRODUCT]](%[[T2]])[%[[T1]]] +// CHECK: affine.for %[[IV:.*]] = 0 to %[[T3]] +// CHECK-DAG: %[[K:.*]] = affine.apply #[[MOD]](%[[IV]])[%[[T1]]] +// CHECK-DAG: %[[T5:.*]] = affine.apply #[[DIV]](%[[IV]])[%[[T1]]] +// CHECK-DAG: %[[J:.*]] = affine.apply #[[MOD]](%[[T5]])[%[[T1]]] +// CHECK-DAG: %[[I:.*]] = affine.apply #[[DIV]](%[[T5]])[%[[T1]]] // CHECK-NEXT: "test.foo"(%[[I]], %[[J]], %[[K]]) // CHECK-NEXT: } // CHECK-NEXT: return @@ -342,12 +333,14 @@ func.func @coalesce_affine_for(%arg0: memref<?x?xf32>) { func.func @test_loops_do_not_get_coalesced() { affine.for %i = 0 to 7 { affine.for %j = #map0(%i) to min #map1(%i) { + "use"(%i, %j) : (index, index) -> () } } return } // CHECK: affine.for %[[IV0:.*]] = 0 to 7 // CHECK-NEXT: affine.for %[[IV1:.*]] = #[[MAP0]](%[[IV0]]) to min #[[MAP1]](%[[IV0]]) +// CHECK-NEXT: "use"(%[[IV0]], %[[IV1]]) // CHECK-NEXT: } // CHECK-NEXT: } // CHECK-NEXT: return diff --git a/mlir/test/Dialect/Affine/loop-fusion-dependence-check.mlir b/mlir/test/Dialect/Affine/loop-fusion-dependence-check.mlir index aa872b0..2c53852 100644 --- a/mlir/test/Dialect/Affine/loop-fusion-dependence-check.mlir +++ b/mlir/test/Dialect/Affine/loop-fusion-dependence-check.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-fusion -test-loop-fusion-dependence-check -split-input-file -verify-diagnostics | FileCheck %s +// RUN: mlir-opt -allow-unregistered-dialect %s -test-loop-fusion=test-loop-fusion-dependence-check -split-input-file -verify-diagnostics | FileCheck %s // ----- diff --git a/mlir/test/Dialect/Affine/loop-fusion-slice-computation.mlir b/mlir/test/Dialect/Affine/loop-fusion-slice-computation.mlir index c303dd0..aa79ee2 100644 --- a/mlir/test/Dialect/Affine/loop-fusion-slice-computation.mlir +++ b/mlir/test/Dialect/Affine/loop-fusion-slice-computation.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -test-loop-fusion -test-loop-fusion-slice-computation -split-input-file -verify-diagnostics | FileCheck %s +// RUN: mlir-opt %s -test-loop-fusion=test-loop-fusion-slice-computation -split-input-file -verify-diagnostics | FileCheck %s // ----- diff --git a/mlir/test/Dialect/Affine/loop-fusion-transformation.mlir b/mlir/test/Dialect/Affine/loop-fusion-transformation.mlir index c8e0918..4f4163a 100644 --- a/mlir/test/Dialect/Affine/loop-fusion-transformation.mlir +++ b/mlir/test/Dialect/Affine/loop-fusion-transformation.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -allow-unregistered-dialect -test-loop-fusion -test-loop-fusion-transformation -split-input-file -canonicalize | FileCheck %s +// RUN: mlir-opt %s -allow-unregistered-dialect -test-loop-fusion=test-loop-fusion-transformation -split-input-file -canonicalize | FileCheck %s // CHECK-LABEL: func @slice_depth1_loop_nest() { func.func @slice_depth1_loop_nest() { diff --git a/mlir/test/Dialect/Affine/slicing-utils.mlir b/mlir/test/Dialect/Affine/slicing-utils.mlir index 71bd8ad..7437997 100644 --- a/mlir/test/Dialect/Affine/slicing-utils.mlir +++ b/mlir/test/Dialect/Affine/slicing-utils.mlir @@ -1,6 +1,6 @@ -// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -affine-super-vectorizer-test -forward-slicing=true 2>&1 | FileCheck %s --check-prefix=FWD -// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -affine-super-vectorizer-test -backward-slicing=true 2>&1 | FileCheck %s --check-prefix=BWD -// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -affine-super-vectorizer-test -slicing=true 2>&1 | FileCheck %s --check-prefix=FWDBWD +// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -affine-super-vectorizer-test="forward-slicing=true" 2>&1 | FileCheck %s --check-prefix=FWD +// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -affine-super-vectorizer-test="backward-slicing=true" 2>&1 | FileCheck %s --check-prefix=BWD +// RUN: mlir-opt -allow-unregistered-dialect %s -split-input-file -affine-super-vectorizer-test="slicing=true" 2>&1 | FileCheck %s --check-prefix=FWDBWD /// 1 2 3 4 /// |_______| |______| diff --git a/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir b/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir index e2be8745..46c4026 100644 --- a/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir +++ b/mlir/test/Dialect/ArmNeon/lower-to-arm-neon.mlir @@ -134,3 +134,127 @@ func.func @test_lower_vector_arm_neon_unroll_incompatible_shape(%lhs: vector<4x1 %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>} %lhs_extsi, %rhs_extsi, %acc : vector<4x12xi32>, vector<4x12xi32> into vector<4x4xi32> return %res : vector<4x4xi32> } + +// ----- + +// CHECK-LABEL: func.func @test_lower_vector_arm_neon_vecmat_unroll( +// CHECK-SAME: %[[VAL_0:.*]]: vector<8xi8>, +// CHECK-SAME: %[[VAL_1:.*]]: vector<8x8xi8>, +// CHECK-SAME: %[[VAL_2:.*]]: vector<8xi32>) -> vector<8xi32> { +// CHECK: %[[VAL_3:.*]] = arith.constant dense<0> : vector<2x2xi32> +// CHECK: %[[VAL_4:.*]] = arith.constant dense<0> : vector<2x8xi8> +// CHECK: %[[VAL_5:.*]] = arith.constant dense<0> : vector<8xi32> +// CHECK: %[[VAL_6:.*]] = vector.extract_strided_slice %[[VAL_1]] {offsets = [0, 0], sizes = [2, 8], strides = [1, 1]} : vector<8x8xi8> to vector<2x8xi8> +// CHECK: %[[VAL_7:.*]] = vector.extract_strided_slice %[[VAL_2]] {offsets = [0], sizes = [2], strides = [1]} : vector<8xi32> to vector<2xi32> +// CHECK: %[[VAL_8:.*]] = vector.insert_strided_slice %[[VAL_0]], %[[VAL_4]] {offsets = [0, 0], strides = [1]} : vector<8xi8> into vector<2x8xi8> +// CHECK: %[[VAL_9:.*]] = vector.insert_strided_slice %[[VAL_7]], %[[VAL_3]] {offsets = [0, 0], strides = [1]} : vector<2xi32> into vector<2x2xi32> +// CHECK: %[[VAL_10:.*]] = vector.shape_cast %[[VAL_8]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_11:.*]] = vector.shape_cast %[[VAL_6]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_12:.*]] = vector.shape_cast %[[VAL_9]] : vector<2x2xi32> to vector<4xi32> +// CHECK: %[[VAL_13:.*]] = arm_neon.intr.smmla %[[VAL_12]], %[[VAL_10]], %[[VAL_11]] : vector<16xi8> to vector<4xi32> +// CHECK: %[[VAL_14:.*]] = vector.shape_cast %[[VAL_13]] : vector<4xi32> to vector<2x2xi32> +// CHECK: %[[VAL_15:.*]] = vector.extract %[[VAL_14]][0] : vector<2xi32> from vector<2x2xi32> +// CHECK: %[[VAL_16:.*]] = vector.insert_strided_slice %[[VAL_15]], %[[VAL_5]] {offsets = [0], strides = [1]} : vector<2xi32> into vector<8xi32> +// CHECK: %[[VAL_17:.*]] = vector.extract_strided_slice %[[VAL_1]] {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<8x8xi8> to vector<2x8xi8> +// CHECK: %[[VAL_18:.*]] = vector.extract_strided_slice %[[VAL_2]] {offsets = [2], sizes = [2], strides = [1]} : vector<8xi32> to vector<2xi32> +// CHECK: %[[VAL_19:.*]] = vector.insert_strided_slice %[[VAL_0]], %[[VAL_4]] {offsets = [0, 0], strides = [1]} : vector<8xi8> into vector<2x8xi8> +// CHECK: %[[VAL_20:.*]] = vector.insert_strided_slice %[[VAL_18]], %[[VAL_3]] {offsets = [0, 0], strides = [1]} : vector<2xi32> into vector<2x2xi32> +// CHECK: %[[VAL_21:.*]] = vector.shape_cast %[[VAL_19]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_22:.*]] = vector.shape_cast %[[VAL_17]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_23:.*]] = vector.shape_cast %[[VAL_20]] : vector<2x2xi32> to vector<4xi32> +// CHECK: %[[VAL_24:.*]] = arm_neon.intr.smmla %[[VAL_23]], %[[VAL_21]], %[[VAL_22]] : vector<16xi8> to vector<4xi32> +// CHECK: %[[VAL_25:.*]] = vector.shape_cast %[[VAL_24]] : vector<4xi32> to vector<2x2xi32> +// CHECK: %[[VAL_26:.*]] = vector.extract %[[VAL_25]][0] : vector<2xi32> from vector<2x2xi32> +// CHECK: %[[VAL_27:.*]] = vector.insert_strided_slice %[[VAL_26]], %[[VAL_16]] {offsets = [2], strides = [1]} : vector<2xi32> into vector<8xi32> +// CHECK: %[[VAL_28:.*]] = vector.extract_strided_slice %[[VAL_1]] {offsets = [4, 0], sizes = [2, 8], strides = [1, 1]} : vector<8x8xi8> to vector<2x8xi8> +// CHECK: %[[VAL_29:.*]] = vector.extract_strided_slice %[[VAL_2]] {offsets = [4], sizes = [2], strides = [1]} : vector<8xi32> to vector<2xi32> +// CHECK: %[[VAL_30:.*]] = vector.insert_strided_slice %[[VAL_0]], %[[VAL_4]] {offsets = [0, 0], strides = [1]} : vector<8xi8> into vector<2x8xi8> +// CHECK: %[[VAL_31:.*]] = vector.insert_strided_slice %[[VAL_29]], %[[VAL_3]] {offsets = [0, 0], strides = [1]} : vector<2xi32> into vector<2x2xi32> +// CHECK: %[[VAL_32:.*]] = vector.shape_cast %[[VAL_30]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_33:.*]] = vector.shape_cast %[[VAL_28]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_34:.*]] = vector.shape_cast %[[VAL_31]] : vector<2x2xi32> to vector<4xi32> +// CHECK: %[[VAL_35:.*]] = arm_neon.intr.smmla %[[VAL_34]], %[[VAL_32]], %[[VAL_33]] : vector<16xi8> to vector<4xi32> +// CHECK: %[[VAL_36:.*]] = vector.shape_cast %[[VAL_35]] : vector<4xi32> to vector<2x2xi32> +// CHECK: %[[VAL_37:.*]] = vector.extract %[[VAL_36]][0] : vector<2xi32> from vector<2x2xi32> +// CHECK: %[[VAL_38:.*]] = vector.insert_strided_slice %[[VAL_37]], %[[VAL_27]] {offsets = [4], strides = [1]} : vector<2xi32> into vector<8xi32> +// CHECK: %[[VAL_39:.*]] = vector.extract_strided_slice %[[VAL_1]] {offsets = [6, 0], sizes = [2, 8], strides = [1, 1]} : vector<8x8xi8> to vector<2x8xi8> +// CHECK: %[[VAL_40:.*]] = vector.extract_strided_slice %[[VAL_2]] {offsets = [6], sizes = [2], strides = [1]} : vector<8xi32> to vector<2xi32> +// CHECK: %[[VAL_41:.*]] = vector.insert_strided_slice %[[VAL_0]], %[[VAL_4]] {offsets = [0, 0], strides = [1]} : vector<8xi8> into vector<2x8xi8> +// CHECK: %[[VAL_42:.*]] = vector.insert_strided_slice %[[VAL_40]], %[[VAL_3]] {offsets = [0, 0], strides = [1]} : vector<2xi32> into vector<2x2xi32> +// CHECK: %[[VAL_43:.*]] = vector.shape_cast %[[VAL_41]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_44:.*]] = vector.shape_cast %[[VAL_39]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_45:.*]] = vector.shape_cast %[[VAL_42]] : vector<2x2xi32> to vector<4xi32> +// CHECK: %[[VAL_46:.*]] = arm_neon.intr.smmla %[[VAL_45]], %[[VAL_43]], %[[VAL_44]] : vector<16xi8> to vector<4xi32> +// CHECK: %[[VAL_47:.*]] = vector.shape_cast %[[VAL_46]] : vector<4xi32> to vector<2x2xi32> +// CHECK: %[[VAL_48:.*]] = vector.extract %[[VAL_47]][0] : vector<2xi32> from vector<2x2xi32> +// CHECK: %[[VAL_49:.*]] = vector.insert_strided_slice %[[VAL_48]], %[[VAL_38]] {offsets = [6], strides = [1]} : vector<2xi32> into vector<8xi32> +// CHECK: return %[[VAL_49]] : vector<8xi32> +// CHECK: } +func.func @test_lower_vector_arm_neon_vecmat_unroll(%lhs: vector<8xi8>, %rhs: vector<8x8xi8>, %acc : vector<8xi32>) -> vector<8xi32> { + %lhs_extsi= arith.extsi %lhs : vector<8xi8> to vector<8xi32> + %rhs_extsi = arith.extsi %rhs : vector<8x8xi8> to vector<8x8xi32> + %res = vector.contract {indexing_maps = [affine_map<(d0, d1) -> (d1)>, affine_map<(d0, d1) -> (d0, d1)>, affine_map<(d0, d1) -> (d0)>], iterator_types = ["parallel", "reduction"], kind = #vector.kind<add>} %lhs_extsi, %rhs_extsi, %acc : vector<8xi32>, vector<8x8xi32> into vector<8xi32> + return %res : vector<8xi32> +} + +// ----- + +// CHECK-LABEL: func.func @test_lower_vector_arm_neon_vecmat_unroll_leading_dim( +// CHECK-SAME: %[[VAL_0:.*]]: vector<1x8xi8>, +// CHECK-SAME: %[[VAL_1:.*]]: vector<8x8xi8>, +// CHECK-SAME: %[[VAL_2:.*]]: vector<1x8xi32>) -> vector<1x8xi32> { +// CHECK: %[[VAL_3:.*]] = arith.constant dense<0> : vector<2x2xi32> +// CHECK: %[[VAL_4:.*]] = arith.constant dense<0> : vector<2x8xi8> +// CHECK: %[[VAL_5:.*]] = arith.constant dense<0> : vector<1x8xi32> +// CHECK: %[[VAL_6:.*]] = vector.extract_strided_slice %[[VAL_1]] {offsets = [0, 0], sizes = [2, 8], strides = [1, 1]} : vector<8x8xi8> to vector<2x8xi8> +// CHECK: %[[VAL_7:.*]] = vector.extract_strided_slice %[[VAL_2]] {offsets = [0, 0], sizes = [1, 2], strides = [1, 1]} : vector<1x8xi32> to vector<1x2xi32> +// CHECK: %[[VAL_8:.*]] = vector.insert_strided_slice %[[VAL_0]], %[[VAL_4]] {offsets = [0, 0], strides = [1, 1]} : vector<1x8xi8> into vector<2x8xi8> +// CHECK: %[[VAL_9:.*]] = vector.insert_strided_slice %[[VAL_7]], %[[VAL_3]] {offsets = [0, 0], strides = [1, 1]} : vector<1x2xi32> into vector<2x2xi32> +// CHECK: %[[VAL_10:.*]] = vector.shape_cast %[[VAL_8]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_11:.*]] = vector.shape_cast %[[VAL_6]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_12:.*]] = vector.shape_cast %[[VAL_9]] : vector<2x2xi32> to vector<4xi32> +// CHECK: %[[VAL_13:.*]] = arm_neon.intr.smmla %[[VAL_12]], %[[VAL_10]], %[[VAL_11]] : vector<16xi8> to vector<4xi32> +// CHECK: %[[VAL_14:.*]] = vector.shape_cast %[[VAL_13]] : vector<4xi32> to vector<2x2xi32> +// CHECK: %[[VAL_15:.*]] = vector.extract %[[VAL_14]][0] : vector<2xi32> from vector<2x2xi32> +// CHECK: %[[VAL_16:.*]] = vector.insert_strided_slice %[[VAL_15]], %[[VAL_5]] {offsets = [0, 0], strides = [1]} : vector<2xi32> into vector<1x8xi32> +// CHECK: %[[VAL_17:.*]] = vector.extract_strided_slice %[[VAL_1]] {offsets = [2, 0], sizes = [2, 8], strides = [1, 1]} : vector<8x8xi8> to vector<2x8xi8> +// CHECK: %[[VAL_18:.*]] = vector.extract_strided_slice %[[VAL_2]] {offsets = [0, 2], sizes = [1, 2], strides = [1, 1]} : vector<1x8xi32> to vector<1x2xi32> +// CHECK: %[[VAL_19:.*]] = vector.insert_strided_slice %[[VAL_0]], %[[VAL_4]] {offsets = [0, 0], strides = [1, 1]} : vector<1x8xi8> into vector<2x8xi8> +// CHECK: %[[VAL_20:.*]] = vector.insert_strided_slice %[[VAL_18]], %[[VAL_3]] {offsets = [0, 0], strides = [1, 1]} : vector<1x2xi32> into vector<2x2xi32> +// CHECK: %[[VAL_21:.*]] = vector.shape_cast %[[VAL_19]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_22:.*]] = vector.shape_cast %[[VAL_17]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_23:.*]] = vector.shape_cast %[[VAL_20]] : vector<2x2xi32> to vector<4xi32> +// CHECK: %[[VAL_24:.*]] = arm_neon.intr.smmla %[[VAL_23]], %[[VAL_21]], %[[VAL_22]] : vector<16xi8> to vector<4xi32> +// CHECK: %[[VAL_25:.*]] = vector.shape_cast %[[VAL_24]] : vector<4xi32> to vector<2x2xi32> +// CHECK: %[[VAL_26:.*]] = vector.extract %[[VAL_25]][0] : vector<2xi32> from vector<2x2xi32> +// CHECK: %[[VAL_27:.*]] = vector.insert_strided_slice %[[VAL_26]], %[[VAL_16]] {offsets = [0, 2], strides = [1]} : vector<2xi32> into vector<1x8xi32> +// CHECK: %[[VAL_28:.*]] = vector.extract_strided_slice %[[VAL_1]] {offsets = [4, 0], sizes = [2, 8], strides = [1, 1]} : vector<8x8xi8> to vector<2x8xi8> +// CHECK: %[[VAL_29:.*]] = vector.extract_strided_slice %[[VAL_2]] {offsets = [0, 4], sizes = [1, 2], strides = [1, 1]} : vector<1x8xi32> to vector<1x2xi32> +// CHECK: %[[VAL_30:.*]] = vector.insert_strided_slice %[[VAL_0]], %[[VAL_4]] {offsets = [0, 0], strides = [1, 1]} : vector<1x8xi8> into vector<2x8xi8> +// CHECK: %[[VAL_31:.*]] = vector.insert_strided_slice %[[VAL_29]], %[[VAL_3]] {offsets = [0, 0], strides = [1, 1]} : vector<1x2xi32> into vector<2x2xi32> +// CHECK: %[[VAL_32:.*]] = vector.shape_cast %[[VAL_30]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_33:.*]] = vector.shape_cast %[[VAL_28]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_34:.*]] = vector.shape_cast %[[VAL_31]] : vector<2x2xi32> to vector<4xi32> +// CHECK: %[[VAL_35:.*]] = arm_neon.intr.smmla %[[VAL_34]], %[[VAL_32]], %[[VAL_33]] : vector<16xi8> to vector<4xi32> +// CHECK: %[[VAL_36:.*]] = vector.shape_cast %[[VAL_35]] : vector<4xi32> to vector<2x2xi32> +// CHECK: %[[VAL_37:.*]] = vector.extract %[[VAL_36]][0] : vector<2xi32> from vector<2x2xi32> +// CHECK: %[[VAL_38:.*]] = vector.insert_strided_slice %[[VAL_37]], %[[VAL_27]] {offsets = [0, 4], strides = [1]} : vector<2xi32> into vector<1x8xi32> +// CHECK: %[[VAL_39:.*]] = vector.extract_strided_slice %[[VAL_1]] {offsets = [6, 0], sizes = [2, 8], strides = [1, 1]} : vector<8x8xi8> to vector<2x8xi8> +// CHECK: %[[VAL_40:.*]] = vector.extract_strided_slice %[[VAL_2]] {offsets = [0, 6], sizes = [1, 2], strides = [1, 1]} : vector<1x8xi32> to vector<1x2xi32> +// CHECK: %[[VAL_41:.*]] = vector.insert_strided_slice %[[VAL_0]], %[[VAL_4]] {offsets = [0, 0], strides = [1, 1]} : vector<1x8xi8> into vector<2x8xi8> +// CHECK: %[[VAL_42:.*]] = vector.insert_strided_slice %[[VAL_40]], %[[VAL_3]] {offsets = [0, 0], strides = [1, 1]} : vector<1x2xi32> into vector<2x2xi32> +// CHECK: %[[VAL_43:.*]] = vector.shape_cast %[[VAL_41]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_44:.*]] = vector.shape_cast %[[VAL_39]] : vector<2x8xi8> to vector<16xi8> +// CHECK: %[[VAL_45:.*]] = vector.shape_cast %[[VAL_42]] : vector<2x2xi32> to vector<4xi32> +// CHECK: %[[VAL_46:.*]] = arm_neon.intr.smmla %[[VAL_45]], %[[VAL_43]], %[[VAL_44]] : vector<16xi8> to vector<4xi32> +// CHECK: %[[VAL_47:.*]] = vector.shape_cast %[[VAL_46]] : vector<4xi32> to vector<2x2xi32> +// CHECK: %[[VAL_48:.*]] = vector.extract %[[VAL_47]][0] : vector<2xi32> from vector<2x2xi32> +// CHECK: %[[VAL_49:.*]] = vector.insert_strided_slice %[[VAL_48]], %[[VAL_38]] {offsets = [0, 6], strides = [1]} : vector<2xi32> into vector<1x8xi32> +// CHECK: return %[[VAL_49]] : vector<1x8xi32> +// CHECK: } +func.func @test_lower_vector_arm_neon_vecmat_unroll_leading_dim(%lhs: vector<1x8xi8>, %rhs: vector<8x8xi8>, %acc : vector<1x8xi32>) -> vector<1x8xi32> { + %lhs_extsi= arith.extsi %lhs : vector<1x8xi8> to vector<1x8xi32> + %rhs_extsi = arith.extsi %rhs : vector<8x8xi8> to vector<8x8xi32> + %res = vector.contract {indexing_maps = [affine_map<(d0, d1, d2) -> (d0, d2)>, affine_map<(d0, d1, d2) -> (d1, d2)>, affine_map<(d0, d1, d2) -> (d0, d1)>], iterator_types = ["parallel", "parallel", "reduction"], kind = #vector.kind<add>} %lhs_extsi, %rhs_extsi, %acc : vector<1x8xi32>, vector<8x8xi32> into vector<1x8xi32> + return %res : vector<1x8xi32> +} diff --git a/mlir/test/Dialect/ControlFlow/ops.mlir b/mlir/test/Dialect/ControlFlow/ops.mlir index 8453c2b..c9317c7 100644 --- a/mlir/test/Dialect/ControlFlow/ops.mlir +++ b/mlir/test/Dialect/ControlFlow/ops.mlir @@ -38,3 +38,16 @@ func.func @switch_i64(%flag : i64, %caseOperand : i32) { ^bb3(%bb3arg : i32): return } + +// CHECK-LABEL: func @switch_result_number +func.func @switch_result_number(%arg0: i32) { + %0:2 = "test.op_with_two_results"() : () -> (i32, i32) + cf.switch %arg0 : i32, [ + default: ^bb2, + 0: ^bb1(%0#0 : i32) + ] + ^bb1(%1: i32): + return + ^bb2: + return +} diff --git a/mlir/test/Dialect/EmitC/invalid_ops.mlir b/mlir/test/Dialect/EmitC/invalid_ops.mlir index 22423cf..bbaab0d 100644 --- a/mlir/test/Dialect/EmitC/invalid_ops.mlir +++ b/mlir/test/Dialect/EmitC/invalid_ops.mlir @@ -390,8 +390,48 @@ func.func @logical_or_resulterror(%arg0: i32, %arg1: i32) { // ----- -func.func @test_subscript_indices_mismatch(%arg0: !emitc.array<4x8xf32>, %arg2: index) { - // expected-error @+1 {{'emitc.subscript' op requires number of indices (1) to match the rank of the array type (2)}} - %0 = emitc.subscript %arg0[%arg2] : <4x8xf32>, index +func.func @test_subscript_array_indices_mismatch(%arg0: !emitc.array<4x8xf32>, %arg1: index) { + // expected-error @+1 {{'emitc.subscript' op on array operand requires number of indices (1) to match the rank of the array type (2)}} + %0 = emitc.subscript %arg0[%arg1] : (!emitc.array<4x8xf32>, index) -> f32 + return +} + +// ----- + +func.func @test_subscript_array_index_type_mismatch(%arg0: !emitc.array<4x8xf32>, %arg1: index, %arg2: f32) { + // expected-error @+1 {{'emitc.subscript' op on array operand requires index operand 1 to be integer-like, but got 'f32'}} + %0 = emitc.subscript %arg0[%arg1, %arg2] : (!emitc.array<4x8xf32>, index, f32) -> f32 + return +} + +// ----- + +func.func @test_subscript_array_type_mismatch(%arg0: !emitc.array<4x8xf32>, %arg1: index, %arg2: index) { + // expected-error @+1 {{'emitc.subscript' op on array operand requires element type ('f32') and result type ('i32') to match}} + %0 = emitc.subscript %arg0[%arg1, %arg2] : (!emitc.array<4x8xf32>, index, index) -> i32 + return +} + +// ----- + +func.func @test_subscript_ptr_indices_mismatch(%arg0: !emitc.ptr<f32>, %arg1: index) { + // expected-error @+1 {{'emitc.subscript' op on pointer operand requires one index operand, but got 2}} + %0 = emitc.subscript %arg0[%arg1, %arg1] : (!emitc.ptr<f32>, index, index) -> f32 + return +} + +// ----- + +func.func @test_subscript_ptr_index_type_mismatch(%arg0: !emitc.ptr<f32>, %arg1: f64) { + // expected-error @+1 {{'emitc.subscript' op on pointer operand requires index operand to be integer-like, but got 'f64'}} + %0 = emitc.subscript %arg0[%arg1] : (!emitc.ptr<f32>, f64) -> f32 + return +} + +// ----- + +func.func @test_subscript_ptr_type_mismatch(%arg0: !emitc.ptr<f32>, %arg1: index) { + // expected-error @+1 {{'emitc.subscript' op on pointer operand requires pointee type ('f32') and result type ('f64') to match}} + %0 = emitc.subscript %arg0[%arg1] : (!emitc.ptr<f32>, index) -> f64 return } diff --git a/mlir/test/Dialect/EmitC/ops.mlir b/mlir/test/Dialect/EmitC/ops.mlir index 5f00a29..ace3670 100644 --- a/mlir/test/Dialect/EmitC/ops.mlir +++ b/mlir/test/Dialect/EmitC/ops.mlir @@ -214,6 +214,13 @@ func.func @test_for_not_index_induction(%arg0 : i16, %arg1 : i16, %arg2 : i16) { return } +func.func @test_subscript(%arg0 : !emitc.array<2x3xf32>, %arg1 : !emitc.ptr<i32>, %arg2 : !emitc.opaque<"std::map<char, int>">, %idx0 : index, %idx1 : i32, %idx2 : !emitc.opaque<"char">) { + %0 = emitc.subscript %arg0[%idx0, %idx1] : (!emitc.array<2x3xf32>, index, i32) -> f32 + %1 = emitc.subscript %arg1[%idx0] : (!emitc.ptr<i32>, index) -> i32 + %2 = emitc.subscript %arg2[%idx2] : (!emitc.opaque<"std::map<char, int>">, !emitc.opaque<"char">) -> !emitc.opaque<"int"> + return +} + emitc.verbatim "#ifdef __cplusplus" emitc.verbatim "extern \"C\" {" emitc.verbatim "#endif // __cplusplus" diff --git a/mlir/test/Dialect/LLVMIR/layout.mlir b/mlir/test/Dialect/LLVMIR/layout.mlir index a78fb77..4813089 100644 --- a/mlir/test/Dialect/LLVMIR/layout.mlir +++ b/mlir/test/Dialect/LLVMIR/layout.mlir @@ -6,6 +6,7 @@ module { // CHECK: alignment = 8 // CHECK: alloca_memory_space = 0 // CHECK: bitsize = 64 + // CHECK: endianness = "" // CHECK: global_memory_space = 0 // CHECK: index = 64 // CHECK: preferred = 8 @@ -16,6 +17,7 @@ module { // CHECK: alignment = 8 // CHECK: alloca_memory_space = 0 // CHECK: bitsize = 64 + // CHECK: endianness = "" // CHECK: global_memory_space = 0 // CHECK: index = 64 // CHECK: preferred = 8 @@ -26,6 +28,7 @@ module { // CHECK: alignment = 8 // CHECK: alloca_memory_space = 0 // CHECK: bitsize = 64 + // CHECK: endianness = "" // CHECK: global_memory_space = 0 // CHECK: index = 64 // CHECK: preferred = 8 @@ -43,6 +46,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< #dlti.dl_entry<!llvm.ptr, dense<[32, 32, 64]> : vector<3xi64>>, #dlti.dl_entry<!llvm.ptr<5>, dense<[64, 64, 64]> : vector<3xi64>>, #dlti.dl_entry<!llvm.ptr<4>, dense<[32, 64, 64, 24]> : vector<4xi64>>, + #dlti.dl_entry<"dlti.endianness", "little">, #dlti.dl_entry<"dlti.alloca_memory_space", 5 : ui64>, #dlti.dl_entry<"dlti.global_memory_space", 2 : ui64>, #dlti.dl_entry<"dlti.program_memory_space", 3 : ui64>, @@ -53,6 +57,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // CHECK: alignment = 4 // CHECK: alloca_memory_space = 5 // CHECK: bitsize = 32 + // CHECK: endianness = "little" // CHECK: global_memory_space = 2 // CHECK: index = 32 // CHECK: preferred = 8 @@ -63,6 +68,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // CHECK: alignment = 4 // CHECK: alloca_memory_space = 5 // CHECK: bitsize = 32 + // CHECK: endianness = "little" // CHECK: global_memory_space = 2 // CHECK: index = 32 // CHECK: preferred = 8 @@ -73,6 +79,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // CHECK: alignment = 8 // CHECK: alloca_memory_space = 5 // CHECK: bitsize = 64 + // CHECK: endianness = "little" // CHECK: global_memory_space = 2 // CHECK: index = 64 // CHECK: preferred = 8 @@ -83,6 +90,7 @@ module attributes { dlti.dl_spec = #dlti.dl_spec< // CHECK: alignment = 8 // CHECK: alloca_memory_space = 5 // CHECK: bitsize = 32 + // CHECK: endianness = "little" // CHECK: global_memory_space = 2 // CHECK: index = 24 // CHECK: preferred = 8 diff --git a/mlir/test/Dialect/LLVMIR/mem2reg-dbginfo.mlir b/mlir/test/Dialect/LLVMIR/mem2reg-dbginfo.mlir index f7ddb4a..b7cbd78 100644 --- a/mlir/test/Dialect/LLVMIR/mem2reg-dbginfo.mlir +++ b/mlir/test/Dialect/LLVMIR/mem2reg-dbginfo.mlir @@ -29,6 +29,27 @@ llvm.func @basic_store_load(%arg0: i64) -> i64 { llvm.return %2 : i64 } +// CHECK-LABEL: llvm.func @multiple_store_load +llvm.func @multiple_store_load(%arg0: i64) -> i64 { + %0 = llvm.mlir.constant(1 : i32) : i32 + // CHECK-NOT: = llvm.alloca + %1 = llvm.alloca %0 x i64 {alignment = 8 : i64} : (i32) -> !llvm.ptr + // CHECK-NOT: llvm.intr.dbg.declare + llvm.intr.dbg.declare #di_local_variable = %1 : !llvm.ptr + // CHECK-NOT: llvm.store + llvm.store %arg0, %1 {alignment = 4 : i64} : i64, !llvm.ptr + // CHECK-NOT: llvm.intr.dbg.declare + llvm.intr.dbg.declare #di_local_variable = %1 : !llvm.ptr + // CHECK: llvm.intr.dbg.value #[[$VAR]] = %[[LOADED:.*]] : i64 + // CHECK: llvm.intr.dbg.value #[[$VAR]] = %[[LOADED]] : i64 + // CHECK-NOT: llvm.intr.dbg.value + // CHECK-NOT: llvm.intr.dbg.declare + // CHECK-NOT: llvm.store + %2 = llvm.load %1 {alignment = 4 : i64} : !llvm.ptr -> i64 + // CHECK: llvm.return %[[LOADED]] : i64 + llvm.return %2 : i64 +} + // CHECK-LABEL: llvm.func @block_argument_value // CHECK-SAME: (%[[ARG0:.*]]: i64, {{.*}}) llvm.func @block_argument_value(%arg0: i64, %arg1: i1) -> i64 { diff --git a/mlir/test/Dialect/Math/expand-math.mlir b/mlir/test/Dialect/Math/expand-math.mlir index bfcff27..3d94b55 100644 --- a/mlir/test/Dialect/Math/expand-math.mlir +++ b/mlir/test/Dialect/Math/expand-math.mlir @@ -610,3 +610,51 @@ func.func @math_fpowi_scalar_zero(%0 : f32) -> f32 { // CHECK: return %[[RET]] : f32 // ----- + +// CHECK-LABEL: func.func @math_fpowi_to_powf_tensor +func.func @math_fpowi_to_powf_tensor(%0 : tensor<8xf32>, %1: tensor<8xi32>) -> tensor<8xf32> { + %2 = math.fpowi %0, %1 : tensor<8xf32>, tensor<8xi32> + return %2 : tensor<8xf32> +} +// CHECK-SAME: (%[[ARG0:.*]]: tensor<8xf32>, %[[ARG1:.*]]: tensor<8xi32>) -> tensor<8xf32> { +// CHECK: %[[CSTNEG1:.*]] = arith.constant dense<-1.000000e+00> : tensor<8xf32> +// CHECK: %[[CST2:.*]] = arith.constant dense<2.000000e+00> : tensor<8xf32> +// CHECK: %[[CST0:.*]] = arith.constant dense<0.000000e+00> : tensor<8xf32> +// CHECK: %[[TOFP:.*]] = arith.sitofp %[[ARG1]] : tensor<8xi32> to tensor<8xf32> +// CHECK: %[[SQ:.*]] = arith.mulf %[[ARG0]], %[[ARG0]] : tensor<8xf32> +// CHECK: %[[DIV:.*]] = arith.divf %[[TOFP]], %[[CST2]] : tensor<8xf32> +// CHECK: %[[LG:.*]] = math.log %[[SQ]] : tensor<8xf32> +// CHECK: %[[MUL:.*]] = arith.mulf %[[DIV]], %[[LG]] : tensor<8xf32> +// CHECK: %[[EXP:.*]] = math.exp %[[MUL]] : tensor<8xf32> +// CHECK: %[[MUL1:.*]] = arith.mulf %[[EXP]], %[[CSTNEG1]] : tensor<8xf32> +// CHECK: %[[REM:.*]] = arith.remf %[[TOFP]], %[[CST2]] : tensor<8xf32> +// CHECK: %[[CMPF:.*]] = arith.cmpf olt, %[[ARG0]], %[[CST0]] : tensor<8xf32> +// CHECK: %[[CMPF1:.*]] = arith.cmpf one, %[[REM]], %[[CST0]] : tensor<8xf32> +// CHECK: %[[AND:.*]] = arith.andi %[[CMPF1]], %[[CMPF]] : tensor<8xi1> +// CHECK: %[[SEL:.*]] = arith.select %[[AND]], %[[MUL1]], %[[EXP]] : tensor<8xi1>, tensor<8xf32> +// CHECK: return %[[SEL]] : tensor<8xf32> + +// ----- + +// CHECK-LABEL: func.func @math_fpowi_to_powf_scalar +func.func @math_fpowi_to_powf_scalar(%0 : f32, %1: i64) -> f32 { + %2 = math.fpowi %0, %1 : f32, i64 + return %2 : f32 +} +// CHECK-SAME: (%[[ARG0:.*]]: f32, %[[ARG1:.*]]: i64) -> f32 { +// CHECK: %[[CSTNEG1:.*]] = arith.constant -1.000000e+00 : f32 +// CHECK: %[[CST2:.*]] = arith.constant 2.000000e+00 : f32 +// CHECK: %[[CST0:.*]] = arith.constant 0.000000e+00 : f32 +// CHECK: %[[TOFP:.*]] = arith.sitofp %[[ARG1]] : i64 to f32 +// CHECK: %[[SQ:.*]] = arith.mulf %[[ARG0]], %[[ARG0]] : f32 +// CHECK: %[[DIV:.*]] = arith.divf %[[TOFP]], %[[CST2]] : f32 +// CHECK: %[[LG:.*]] = math.log %[[SQ]] : f32 +// CHECK: %[[MUL:.*]] = arith.mulf %[[DIV]], %[[LG]] : f32 +// CHECK: %[[EXP:.*]] = math.exp %[[MUL]] : f32 +// CHECK: %[[MUL1:.*]] = arith.mulf %[[EXP]], %[[CSTNEG1]] : f32 +// CHECK: %[[REM:.*]] = arith.remf %[[TOFP]], %[[CST2]] : f32 +// CHECK: %[[CMPF:.*]] = arith.cmpf olt, %[[ARG0]], %[[CST0]] : f32 +// CHECK: %[[CMPF1:.*]] = arith.cmpf one, %[[REM]], %[[CST0]] : f32 +// CHECK: %[[AND:.*]] = arith.andi %[[CMPF1]], %[[CMPF]] : i1 +// CHECK: %[[SEL:.*]] = arith.select %[[AND]], %[[MUL1]], %[[EXP]] : f32 +// CHECK: return %[[SEL]] : f32 diff --git a/mlir/test/Dialect/OpenMP/invalid.mlir b/mlir/test/Dialect/OpenMP/invalid.mlir index a00383c..1134db7 100644 --- a/mlir/test/Dialect/OpenMP/invalid.mlir +++ b/mlir/test/Dialect/OpenMP/invalid.mlir @@ -436,6 +436,25 @@ atomic { // ----- +// expected-error @below {{op expects cleanup region with one argument of the reduction type}} +omp.declare_reduction @add_f32 : f32 +init { +^bb0(%arg: f32): + %0 = arith.constant 0.0 : f32 + omp.yield (%0 : f32) +} +combiner { +^bb1(%arg0: f32, %arg1: f32): + %1 = arith.addf %arg0, %arg1 : f32 + omp.yield (%1 : f32) +} +cleanup { +^bb0(%arg: f64): + omp.yield +} + +// ----- + func.func @foo(%lb : index, %ub : index, %step : index) { %c1 = arith.constant 1 : i32 %0 = llvm.alloca %c1 x i32 : (i32) -> !llvm.ptr diff --git a/mlir/test/Dialect/OpenMP/ops.mlir b/mlir/test/Dialect/OpenMP/ops.mlir index 30ce774..e2c255c 100644 --- a/mlir/test/Dialect/OpenMP/ops.mlir +++ b/mlir/test/Dialect/OpenMP/ops.mlir @@ -603,6 +603,8 @@ func.func @omp_target_pretty(%if_cond : i1, %device : si32, %num_threads : i32) // CHECK: atomic // CHECK: ^{{.+}}(%{{.+}}: !llvm.ptr, %{{.+}}: !llvm.ptr): // CHECK: omp.yield +// CHECK: cleanup +// CHECK: omp.yield omp.declare_reduction @add_f32 : f32 init { ^bb0(%arg: f32): @@ -620,6 +622,10 @@ atomic { llvm.atomicrmw fadd %arg2, %2 monotonic : !llvm.ptr, f32 omp.yield } +cleanup { +^bb0(%arg: f32): + omp.yield +} // CHECK-LABEL: func @wsloop_reduction func.func @wsloop_reduction(%lb : index, %ub : index, %step : index) { @@ -789,6 +795,7 @@ combiner { omp.yield (%1 : f32) } // CHECK-NOT: atomic +// CHECK-NOT: cleanup // CHECK-LABEL: func @wsloop_reduction2 func.func @wsloop_reduction2(%lb : index, %ub : index, %step : index) { @@ -2088,6 +2095,7 @@ func.func @opaque_pointers_atomic_rwu(%v: !llvm.ptr, %x: !llvm.ptr) { // CHECK-LABEL: @opaque_pointers_reduction // CHECK: atomic { // CHECK-NEXT: ^{{[[:alnum:]]+}}(%{{.*}}: !llvm.ptr, %{{.*}}: !llvm.ptr): +// CHECK-NOT: cleanup omp.declare_reduction @opaque_pointers_reduction : f32 init { ^bb0(%arg: f32): diff --git a/mlir/test/Dialect/SCF/transform-op-coalesce.mlir b/mlir/test/Dialect/SCF/transform-op-coalesce.mlir index 2d59331..4dc3e4e 100644 --- a/mlir/test/Dialect/SCF/transform-op-coalesce.mlir +++ b/mlir/test/Dialect/SCF/transform-op-coalesce.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt %s -transform-interpreter -split-input-file -verify-diagnostics | FileCheck %s +// RUN: mlir-opt %s -transform-interpreter -split-input-file -verify-diagnostics -allow-unregistered-dialect --cse | FileCheck %s func.func @coalesce_inner() { %c0 = arith.constant 0 : index @@ -14,7 +14,7 @@ func.func @coalesce_inner() { scf.for %k = %i to %j step %c1 { // Inner loop must have been removed. scf.for %l = %i to %j step %c1 { - arith.addi %i, %j : index + "use"(%i, %j) : (index, index) -> () } } {coalesce} } @@ -33,13 +33,19 @@ module attributes {transform.with_named_sequence} { // ----- +// CHECK-DAG: #[[MAP:.+]] = affine_map<() -> (64)> +// CHECK-DAG: #[[MAP1:.+]] = affine_map<(d0)[s0] -> (d0 * s0)> +// CHECK-DAG: #[[MAP2:.+]] = affine_map<(d0)[s0] -> (d0 mod s0)> +// CHECK-DAG: #[[MAP3:.+]] = affine_map<(d0)[s0] -> (d0 floordiv s0)> func.func @coalesce_outer(%arg1: memref<64x64xf32, 1>, %arg2: memref<64x64xf32, 1>, %arg3: memref<64x64xf32, 1>) attributes {} { + // CHECK: %[[T0:.+]] = affine.apply #[[MAP]]() + // CHECK: %[[UB:.+]] = affine.apply #[[MAP1]](%[[T0]])[%[[T0]]] // CHECK: affine.for %[[IV1:.+]] = 0 to %[[UB:.+]] { // CHECK-NOT: affine.for %[[IV2:.+]] affine.for %arg4 = 0 to 64 { affine.for %arg5 = 0 to 64 { - // CHECK: %[[IDX0:.+]] = affine.apply #[[MAP0:.+]](%[[IV1]])[%{{.+}}] - // CHECK: %[[IDX1:.+]] = affine.apply #[[MAP1:.+]](%[[IV1]])[%{{.+}}] + // CHECK: %[[IDX0:.+]] = affine.apply #[[MAP2]](%[[IV1]])[%{{.+}}] + // CHECK: %[[IDX1:.+]] = affine.apply #[[MAP3]](%[[IV1]])[%{{.+}}] // CHECK-NEXT: %{{.+}} = affine.load %{{.+}}[%[[IDX1]], %[[IDX0]]] : memref<64x64xf32, 1> %0 = affine.load %arg1[%arg4, %arg5] : memref<64x64xf32, 1> %1 = affine.load %arg2[%arg4, %arg5] : memref<64x64xf32, 1> @@ -96,3 +102,200 @@ module attributes {transform.with_named_sequence} { transform.yield } } + +// ----- + +func.func @tensor_loops(%arg0 : tensor<?x?xf32>, %lb0 : index, %ub0 : index, %step0 : index, + %lb1 : index, %ub1 : index, %step1 : index, %lb2 : index, %ub2 : index, %step2 : index) -> tensor<?x?xf32> { + %0 = scf.for %i = %lb0 to %ub0 step %step0 iter_args(%arg1 = %arg0) -> tensor<?x?xf32> { + %1 = scf.for %j = %lb1 to %ub1 step %step1 iter_args(%arg2 = %arg1) -> tensor<?x?xf32> { + %2 = scf.for %k = %lb2 to %ub2 step %step2 iter_args(%arg3 = %arg2) -> tensor<?x?xf32> { + %3 = "use"(%arg3, %i, %j, %k) : (tensor<?x?xf32>, index, index, index) -> (tensor<?x?xf32>) + scf.yield %3 : tensor<?x?xf32> + } + scf.yield %2 : tensor<?x?xf32> + } + scf.yield %1 : tensor<?x?xf32> + } {coalesce} + return %0 : tensor<?x?xf32> +} +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["scf.for"]} attributes {coalesce} in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = transform.cast %0 : !transform.any_op to !transform.op<"scf.for"> + %2 = transform.loop.coalesce %1 : (!transform.op<"scf.for">) -> (!transform.op<"scf.for">) + transform.yield + } +} +// CHECK: func.func @tensor_loops( +// CHECK-SAME: %[[ARG0:.+]]: tensor<?x?xf32> +// CHECK-SAME: %[[LB0:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[UB0:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[STEP0:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[LB1:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[UB1:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[STEP1:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[LB2:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[UB2:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[STEP2:[a-zA-Z0-9_]+]]: index +// CHECK: %[[NEWUB0_DIFF:.+]] = arith.subi %[[UB0]], %[[LB0]] +// CHECK-DAG: %[[NEWUB0:.+]] = arith.ceildivsi %[[NEWUB0_DIFF]], %[[STEP0]] +// CHECK-DAG: %[[C0:.+]] = arith.constant 0 +// CHECK-DAG: %[[C1:.+]] = arith.constant 1 +// CHECK: %[[NEWUB1_DIFF:.+]] = arith.subi %[[UB1]], %[[LB1]] +// CHECK-DAG: %[[NEWUB1:.+]] = arith.ceildivsi %[[NEWUB1_DIFF]], %[[STEP1]] +// CHECK: %[[NEWUB2_DIFF:.+]] = arith.subi %[[UB2]], %[[LB2]] +// CHECK-DAG: %[[NEWUB2:.+]] = arith.ceildivsi %[[NEWUB2_DIFF]], %[[STEP2]] +// CHECK: %[[PROD1:.+]] = arith.muli %[[NEWUB0]], %[[NEWUB1]] +// CHECK: %[[NEWUB:.+]] = arith.muli %[[PROD1]], %[[NEWUB2]] +// CHECK: %[[RESULT:.+]] = scf.for %[[IV:[a-zA-Z0-9]+]] = %[[C0]] to %[[NEWUB]] step %[[C1]] iter_args(%[[ITER_ARG:.+]] = %[[ARG0]]) +// CHECK: %[[IV2:.+]] = arith.remsi %[[IV]], %[[NEWUB2]] +// CHECK: %[[PREVIOUS:.+]] = arith.divsi %[[IV]], %[[NEWUB2]] +// CHECK: %[[IV1:.+]] = arith.remsi %[[PREVIOUS]], %[[NEWUB1]] +// CHECK: %[[IV0:.+]] = arith.divsi %[[PREVIOUS]], %[[NEWUB1]] +// CHECK: %[[K_STEP:.+]] = arith.muli %[[IV2]], %[[STEP2]] +// CHECK: %[[K:.+]] = arith.addi %[[K_STEP]], %[[LB2]] +// CHECK: %[[J_STEP:.+]] = arith.muli %[[IV1]], %[[STEP1]] +// CHECK: %[[J:.+]] = arith.addi %[[J_STEP]], %[[LB1]] +// CHECK: %[[I_STEP:.+]] = arith.muli %[[IV0]], %[[STEP0]] +// CHECK: %[[I:.+]] = arith.addi %[[I_STEP]], %[[LB0]] +// CHECK: %[[USE:.+]] = "use"(%[[ITER_ARG]], %[[I]], %[[J]], %[[K]]) +// CHECK: scf.yield %[[USE]] +// CHECK: return %[[RESULT]] + +// ----- + +// Coalesce only first two loops, but not the last since the iter_args dont line up +func.func @tensor_loops_first_two(%arg0 : tensor<?x?xf32>, %arg1 : tensor<?x?xf32>, %lb0 : index, %ub0 : index, %step0 : index, + %lb1 : index, %ub1 : index, %step1 : index, %lb2 : index, %ub2 : index, %step2 : index) -> (tensor<?x?xf32>, tensor<?x?xf32>) { + %0:2 = scf.for %i = %lb0 to %ub0 step %step0 iter_args(%arg2 = %arg0, %arg3 = %arg1) -> (tensor<?x?xf32>, tensor<?x?xf32>) { + %1:2 = scf.for %j = %lb1 to %ub1 step %step1 iter_args(%arg4 = %arg2, %arg5 = %arg3) -> (tensor<?x?xf32>, tensor<?x?xf32>) { + %2:2 = scf.for %k = %lb2 to %ub2 step %step2 iter_args(%arg6 = %arg5, %arg7 = %arg4) -> (tensor<?x?xf32>, tensor<?x?xf32>) { + %3:2 = "use"(%arg3, %i, %j, %k) : (tensor<?x?xf32>, index, index, index) -> (tensor<?x?xf32>, tensor<?x?xf32>) + scf.yield %3#0, %3#1 : tensor<?x?xf32>, tensor<?x?xf32> + } + scf.yield %2#0, %2#1 : tensor<?x?xf32>, tensor<?x?xf32> + } + scf.yield %1#0, %1#1 : tensor<?x?xf32>, tensor<?x?xf32> + } {coalesce} + return %0#0, %0#1 : tensor<?x?xf32>, tensor<?x?xf32> +} +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["scf.for"]} attributes {coalesce} in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = transform.cast %0 : !transform.any_op to !transform.op<"scf.for"> + %2 = transform.loop.coalesce %1 : (!transform.op<"scf.for">) -> (!transform.op<"scf.for">) + transform.yield + } +} +// CHECK: func.func @tensor_loops_first_two( +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]]: tensor<?x?xf32> +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]]: tensor<?x?xf32> +// CHECK-SAME: %[[LB0:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[UB0:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[STEP0:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[LB1:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[UB1:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[STEP1:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[LB2:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[UB2:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[STEP2:[a-zA-Z0-9_]+]]: index +// CHECK: scf.for +// CHECK: arith.remsi +// CHECK: arith.divsi +// CHECK: scf.for %{{[a-zA-Z0-9]+}} = %[[LB2]] to %[[UB2]] step %[[STEP2]] +// CHECK-NOT: scf.for +// CHECK: transform.named_sequence + +// ----- + +// Coalesce only first two loops, but not the last since the yields dont match up +func.func @tensor_loops_first_two_2(%arg0 : tensor<?x?xf32>, %arg1 : tensor<?x?xf32>, %lb0 : index, %ub0 : index, %step0 : index, + %lb1 : index, %ub1 : index, %step1 : index, %lb2 : index, %ub2 : index, %step2 : index) -> (tensor<?x?xf32>, tensor<?x?xf32>) { + %0:2 = scf.for %i = %lb0 to %ub0 step %step0 iter_args(%arg2 = %arg0, %arg3 = %arg1) -> (tensor<?x?xf32>, tensor<?x?xf32>) { + %1:2 = scf.for %j = %lb1 to %ub1 step %step1 iter_args(%arg4 = %arg2, %arg5 = %arg3) -> (tensor<?x?xf32>, tensor<?x?xf32>) { + %2:2 = scf.for %k = %lb2 to %ub2 step %step2 iter_args(%arg6 = %arg4, %arg7 = %arg5) -> (tensor<?x?xf32>, tensor<?x?xf32>) { + %3:2 = "use"(%arg3, %i, %j, %k) : (tensor<?x?xf32>, index, index, index) -> (tensor<?x?xf32>, tensor<?x?xf32>) + scf.yield %3#0, %3#1 : tensor<?x?xf32>, tensor<?x?xf32> + } + scf.yield %2#1, %2#0 : tensor<?x?xf32>, tensor<?x?xf32> + } + scf.yield %1#0, %1#1 : tensor<?x?xf32>, tensor<?x?xf32> + } {coalesce} + return %0#0, %0#1 : tensor<?x?xf32>, tensor<?x?xf32> +} +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["scf.for"]} attributes {coalesce} in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = transform.cast %0 : !transform.any_op to !transform.op<"scf.for"> + %2 = transform.loop.coalesce %1 : (!transform.op<"scf.for">) -> (!transform.op<"scf.for">) + transform.yield + } +} +// CHECK: func.func @tensor_loops_first_two_2( +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]]: tensor<?x?xf32> +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]]: tensor<?x?xf32> +// CHECK-SAME: %[[LB0:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[UB0:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[STEP0:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[LB1:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[UB1:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[STEP1:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[LB2:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[UB2:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[STEP2:[a-zA-Z0-9_]+]]: index +// CHECK: scf.for +// CHECK: arith.remsi +// CHECK: arith.divsi +// CHECK: scf.for %{{[a-zA-Z0-9]+}} = %[[LB2]] to %[[UB2]] step %[[STEP2]] +// CHECK-NOT: scf.for +// CHECK: transform.named_sequence + +// ----- + +// Coalesce only last two loops, but not the first since the yields dont match up +func.func @tensor_loops_last_two(%arg0 : tensor<?x?xf32>, %arg1 : tensor<?x?xf32>, %lb0 : index, %ub0 : index, %step0 : index, + %lb1 : index, %ub1 : index, %step1 : index, %lb2 : index, %ub2 : index, %step2 : index) -> (tensor<?x?xf32>, tensor<?x?xf32>) { + %0:2 = scf.for %i = %lb0 to %ub0 step %step0 iter_args(%arg2 = %arg0, %arg3 = %arg1) -> (tensor<?x?xf32>, tensor<?x?xf32>) { + %1:2 = scf.for %j = %lb1 to %ub1 step %step1 iter_args(%arg4 = %arg2, %arg5 = %arg3) -> (tensor<?x?xf32>, tensor<?x?xf32>) { + %2:2 = scf.for %k = %lb2 to %ub2 step %step2 iter_args(%arg6 = %arg4, %arg7 = %arg5) -> (tensor<?x?xf32>, tensor<?x?xf32>) { + %3:2 = "use"(%arg3, %i, %j, %k) : (tensor<?x?xf32>, index, index, index) -> (tensor<?x?xf32>, tensor<?x?xf32>) + scf.yield %3#0, %3#1 : tensor<?x?xf32>, tensor<?x?xf32> + } + scf.yield %2#0, %2#1 : tensor<?x?xf32>, tensor<?x?xf32> + } + scf.yield %1#1, %1#0 : tensor<?x?xf32>, tensor<?x?xf32> + } {coalesce} + return %0#0, %0#1 : tensor<?x?xf32>, tensor<?x?xf32> +} +module attributes {transform.with_named_sequence} { + transform.named_sequence @__transform_main(%arg1: !transform.any_op {transform.readonly}) { + %0 = transform.structured.match ops{["scf.for"]} attributes {coalesce} in %arg1 : (!transform.any_op) -> !transform.any_op + %1 = transform.cast %0 : !transform.any_op to !transform.op<"scf.for"> + %2 = transform.loop.coalesce %1 : (!transform.op<"scf.for">) -> (!transform.op<"scf.for">) + transform.yield + } +} +// CHECK: func.func @tensor_loops_last_two( +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]]: tensor<?x?xf32> +// CHECK-SAME: %[[ARG0:[a-zA-Z0-9]+]]: tensor<?x?xf32> +// CHECK-SAME: %[[LB0:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[UB0:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[STEP0:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[LB1:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[UB1:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[STEP1:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[LB2:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[UB2:[a-zA-Z0-9_]+]]: index +// CHECK-SAME: %[[STEP2:[a-zA-Z0-9_]+]]: index +// CHECK: scf.for %{{[a-zA-Z0-9]+}} = %[[LB0]] to %[[UB0]] step %[[STEP0]] +// CHECK: arith.subi +// CHECK: arith.ceildivsi +// CHECK: arith.subi +// CHECK: arith.ceildivsi +// CHECK: scf.for +// CHECK: arith.remsi +// CHECK: arith.divsi +// CHECK-NOT: scf.for +// CHECK: transform.named_sequence + diff --git a/mlir/test/Dialect/SparseTensor/one_shot_bufferize_tensor_copy_insertion.mlir b/mlir/test/Dialect/SparseTensor/one_shot_bufferize_tensor_copy_insertion.mlir index 6c2292b..b769acd 100644 --- a/mlir/test/Dialect/SparseTensor/one_shot_bufferize_tensor_copy_insertion.mlir +++ b/mlir/test/Dialect/SparseTensor/one_shot_bufferize_tensor_copy_insertion.mlir @@ -70,3 +70,39 @@ func.func @update_notinplace(%argb: tensor<10xf32>, %arga: tensor<10xf32, #SV>) } -> tensor<10xf32> return %0, %argb : tensor<10xf32>, tensor<10xf32> } + +#map = affine_map<(d0, d1) -> (d0, d1)> +#map1 = affine_map<(d0, d1, d2) -> (d0, d2)> +#map2 = affine_map<(d0, d1, d2) -> (d2, d1)> +#map3 = affine_map<(d0, d1, d2) -> (d0, d1)> +#sparse = #sparse_tensor.encoding<{ map = (d0, d1) -> (d0 : dense, d1 : compressed), posWidth = 64, crdWidth = 64 }> + +// linalg.generic with sparse tensors does not necessarily bufferize to +// element-wise access into the underlying sparse data structures. + +// CHECK-LABEL: func @sparse_non_elementwise( +func.func @sparse_non_elementwise(%arg0: tensor<64x64xf32, #sparse>, %arg1: tensor<64x64xf32>, %arg2: tensor<64x64xf32>) -> tensor<64x64xf32> { + %cst = arith.constant 0.000000e+00 : f32 + // CHECK: %[[alloc0:.*]] = bufferization.alloc_tensor() + // CHECK: %[[alloc1:.*]] = bufferization.alloc_tensor() + %0 = bufferization.alloc_tensor() : tensor<64x64xf32> + // CHECK: %[[generic0:.*]] = linalg.generic {{.*}} outs(%[[alloc1]] : {{.*}}) + %1 = linalg.generic {indexing_maps = [#map], iterator_types = ["parallel", "parallel"]} outs(%0 : tensor<64x64xf32>) { + ^bb0(%out: f32): + linalg.yield %cst : f32 + } -> tensor<64x64xf32> + // CHECK: linalg.generic {{.*}} outs(%[[generic0]] : {{.*}}) + %2 = linalg.generic {indexing_maps = [#map1, #map2, #map3], iterator_types = ["parallel", "parallel", "reduction"]} ins(%arg2, %arg2 : tensor<64x64xf32>, tensor<64x64xf32>) outs(%1 : tensor<64x64xf32>) { + ^bb0(%in: f32, %in_0: f32, %out: f32): + %4 = arith.mulf %in, %in_0 : f32 + %5 = arith.addf %out, %4 : f32 + linalg.yield %5 : f32 + } -> tensor<64x64xf32> + // CHECK: linalg.generic {{.*}} outs(%[[alloc0]] : {{.*}}) + %3 = linalg.generic {indexing_maps = [#map, #map, #map], iterator_types = ["parallel", "parallel"]} ins(%arg0, %2 : tensor<64x64xf32, #sparse>, tensor<64x64xf32>) outs(%0 : tensor<64x64xf32>) attrs = {sorted = true} { + ^bb0(%in: f32, %in_0: f32, %out: f32): + %4 = arith.mulf %in, %in_0 : f32 + linalg.yield %4 : f32 + } -> tensor<64x64xf32> + return %3 : tensor<64x64xf32> +} diff --git a/mlir/test/Dialect/Tosa/invalid.mlir b/mlir/test/Dialect/Tosa/invalid.mlir index 38ba48f..730ac41 100644 --- a/mlir/test/Dialect/Tosa/invalid.mlir +++ b/mlir/test/Dialect/Tosa/invalid.mlir @@ -243,38 +243,70 @@ func.func @test_reshape_type_mismatch(%arg0 : tensor<13x21x3xf32>) -> () { // ----- -func.func @test_reverse_axis_out_of_range(%arg0 : tensor<13x21x3xf32>) -> () { - // expected-error@+1 {{'tosa.reverse' op expect input tensor rank (3) to be larger than reverse axis (5)}} - %0 = tosa.reverse %arg0 {axis = 5 : i32} : (tensor<13x21x3xf32>) -> tensor<?x?x?xi32> +func.func @test_reshape_static_zero_dim_input(%arg0 : tensor<13x0x3xf32>) -> () { + // expected-error@+1 {{'tosa.reshape' op tensor has a dimension with size zero. Each dimension of a tensor must have size >= 1}} + %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 13, 21, 3>} : (tensor<13x0x3xf32>) -> tensor<13x0x3xf32> return } // ----- -func.func @test_const_attribute_type_mismatch() -> tensor<100x100xf32> { - // expected-error@+1 {{'tosa.const' op failed to verify that all of {value, output} have same shape}} - %0 = "tosa.const"() {value = dense<0.000000e+00> : tensor<1x1xf32>} : () -> tensor<100x100xf32> - return %0 : tensor<100x100xf32> +func.func @test_reshape_zero_dim_input(%arg0 : tensor<?x0x3xf32>) -> () { + // expected-error@+1 {{'tosa.reshape' op tensor has a dimension with size zero. Each dimension of a tensor must have size >= 1}} + %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 13, 21, 3>} : (tensor<?x0x3xf32>) -> tensor<13x0x3xf32> + return } // ----- -func.func @test_reshape_static_zero_dim_input(%arg0 : tensor<13x0x3xf32>) -> () { - // expected-error@+1 {{'tosa.reshape' op tensor has a dimension with size zero. Each dimension of a tensor must have size >= 1}} - %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 13, 21, 3>} : (tensor<13x0x3xf32>) -> tensor<13x0x3xf32> +func.func @test_reshape_rank_mismatch(%arg0 : tensor<?xf32>) -> () { + // expected-error@+1 {{'tosa.reshape' op new shape does not match result rank}} + %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 4>} : (tensor<?xf32>) -> tensor<?xf32> return } // ----- -func.func @test_reshape_zero_dim_input(%arg0 : tensor<?x0x3xf32>) -> () { - // expected-error@+1 {{'tosa.reshape' op tensor has a dimension with size zero. Each dimension of a tensor must have size >= 1}} - %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 13, 21, 3>} : (tensor<?x0x3xf32>) -> tensor<13x0x3xf32> +func.func @test_reshape_inconsistent_result_type(%arg0 : tensor<?xf32>) -> () { + // expected-error@+1 {{'tosa.reshape' op new shape is inconsistent with result shape}} + %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, 4, -1>} : (tensor<?xf32>) -> tensor<?x3x5xf32> + return +} + +// ----- + +func.func @test_reshape_invalid_size(%arg0 : tensor<2x4xf32>) -> () { + // expected-error@+1 {{'tosa.reshape' op cannot reshape 8 elements into 15}} + %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 3, 5>} : (tensor<2x4xf32>) -> tensor<3x5xf32> + return +} + +// ----- + +func.func @test_reshape_invalid_placeholders(%arg0 : tensor<?xf32>) -> () { + // expected-error@+1 {{'tosa.reshape' op expected at most one target dimension to be -1}} + %0 = "tosa.reshape"(%arg0) {new_shape = array<i64: 2, -1, -1>} : (tensor<?xf32>) -> tensor<2x?x?xf32> return } // ----- +func.func @test_reverse_axis_out_of_range(%arg0 : tensor<13x21x3xf32>) -> () { + // expected-error@+1 {{'tosa.reverse' op expect input tensor rank (3) to be larger than reverse axis (5)}} + %0 = tosa.reverse %arg0 {axis = 5 : i32} : (tensor<13x21x3xf32>) -> tensor<?x?x?xi32> + return +} + +// ----- + +func.func @test_const_attribute_type_mismatch() -> tensor<100x100xf32> { + // expected-error@+1 {{'tosa.const' op failed to verify that all of {value, output} have same shape}} + %0 = "tosa.const"() {value = dense<0.000000e+00> : tensor<1x1xf32>} : () -> tensor<100x100xf32> + return %0 : tensor<100x100xf32> +} + +// ----- + func.func @test_conv2d_static_zero_dim_input(%arg0: tensor<1x29x0x4xf32>, %arg1: tensor<16x3x3x4xf32>, %arg2: tensor<16xf32>) -> tensor<1x27x27x16xf32> { // expected-error@+1 {{'tosa.conv2d' op tensor has a dimension with size zero. Each dimension of a tensor must have size >= 1}} %0 = "tosa.conv2d"(%arg0, %arg1, %arg2) {dilation = array<i64: 1, 1>, pad = array<i64: 0, 0, 0, 0>, stride = array<i64: 1, 1>} diff --git a/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir b/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir index 1f0cfaf..2be1204 100644 --- a/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir +++ b/mlir/test/Dialect/Tosa/tosa-infer-shapes.mlir @@ -1,4 +1,4 @@ -// RUN: mlir-opt --split-input-file --tosa-infer-shapes %s | FileCheck %s +// RUN: mlir-opt --split-input-file --tosa-infer-shapes --allow-unregistered-dialect %s | FileCheck %s // CHECK-LABEL: @test_return func.func @test_return(%arg0 : tensor<4xf32>) -> tensor<*xf32> { @@ -1177,6 +1177,97 @@ func.func @while_test(%arg0 : tensor<i32>, %arg1 : tensor<1xi32>) -> () { // ----- +// This test locks down a fix for a crash in the type inference process. +// The relevant pattern is a while loop whose body contains a TOSA operation which is +// consumed by a non-inferrable user in the same body. +// Previously, this would trigger a crash due to how types are cached and then +// reapplied to the operations in the loops body. + +// CHECK-LABEL: @while_dont_crash +func.func @while_dont_crash(%arg0 : tensor<i32>) -> (tensor<*xi32>) { + %0 = tosa.add %arg0, %arg0 : (tensor<i32>, tensor<i32>) -> tensor<*xi32> + // CHECK: tosa.while_loop + // CHECK-SAME: (tensor<i32>) -> tensor<i32> + %1 = tosa.while_loop (%arg1 = %0) : (tensor<*xi32>) -> tensor<*xi32> { + %2 = "tosa.const"() <{value = dense<3> : tensor<i32>}> : () -> tensor<i32> + // CHECK: tosa.greater_equal + // CHECK-SAME: (tensor<i32>, tensor<i32>) -> tensor<i1> + %3 = tosa.greater_equal %2, %arg1 : (tensor<i32>, tensor<*xi32>) -> tensor<*xi1> + tosa.yield %3 : tensor<*xi1> + } do { + // CHECK: ^bb0 + // CHECK-SAME: tensor<i32> + ^bb0(%arg1: tensor<*xi32>): + // CHECK: tosa.add + // CHECK-SAME: (tensor<i32>, tensor<i32>) -> tensor<i32> + %3 = tosa.add %arg1, %arg1 : (tensor<*xi32>, tensor<*xi32>) -> tensor<*xi32> + // CHECK: %[[CAST:.+]] = tensor.cast %{{.*}} : tensor<i32> to tensor<*xi32> + // CHECK: "use"(%[[CAST]]) : (tensor<*xi32>) -> () + "use"(%3) : (tensor<*xi32>) -> () + tosa.yield %3 : tensor<*xi32> + } + // CHECK: tensor.cast + return %1 : tensor<*xi32> +} + +// ----- + +// This test locks down a fix for a crash in the type inference process. +// The relevant pattern is a while loop whose body contains a TOSA operation which is +// consumed by a non-inferrable user in the same body. + +// CHECK-LABEL: @while_dont_crash_nested +func.func @while_dont_crash_nested(%arg0 : tensor<i32>) -> (tensor<*xi32>) { + %0 = tosa.add %arg0, %arg0 : (tensor<i32>, tensor<i32>) -> tensor<*xi32> + // CHECK: tosa.while_loop + // CHECK-SAME: (tensor<i32>) -> tensor<i32> + %1 = tosa.while_loop (%arg1 = %0) : (tensor<*xi32>) -> tensor<*xi32> { + %2 = "tosa.const"() <{value = dense<3> : tensor<i32>}> : () -> tensor<i32> + // CHECK: tosa.greater_equal + // CHECK-SAME: (tensor<i32>, tensor<i32>) -> tensor<i1> + %3 = tosa.greater_equal %2, %arg1 : (tensor<i32>, tensor<*xi32>) -> tensor<*xi1> + // CHECK: tosa.yield + // CHECK-SAME: tensor<i1> + tosa.yield %3 : tensor<*xi1> + } do { + // CHECK: ^bb0 + // CHECK-SAME: tensor<i32> + ^bb0(%arg1: tensor<*xi32>): + // CHECK: tosa.while_loop + // CHECK-SAME: (tensor<i32>) -> tensor<i32> + %1 = tosa.while_loop (%arg2 = %arg1) : (tensor<*xi32>) -> tensor<*xi32> { + %2 = "tosa.const"() <{value = dense<3> : tensor<i32>}> : () -> tensor<i32> + // CHECK: tosa.greater_equal + // CHECK-SAME: (tensor<i32>, tensor<i32>) -> tensor<i1> + %4 = tosa.greater_equal %2, %arg2 : (tensor<i32>, tensor<*xi32>) -> tensor<*xi1> + // CHECK: tosa.yield + // CHECK-SAME: tensor<i1> + tosa.yield %4 : tensor<*xi1> + } do { + // CHECK: ^bb0 + // CHECK-SAME: tensor<i32> + ^bb0(%arg2: tensor<*xi32>): + // CHECK: tosa.add + // CHECK-SAME: (tensor<i32>, tensor<i32>) -> tensor<i32> + %4 = tosa.add %arg2, %arg2 : (tensor<*xi32>, tensor<*xi32>) -> tensor<*xi32> + // CHECK: %[[CAST:.+]] = tensor.cast %{{.*}} : tensor<i32> to tensor<*xi32> + // CHECK: "use"(%[[CAST]]) : (tensor<*xi32>) -> () + "use"(%4) : (tensor<*xi32>) -> () + // CHECK: tosa.yield + // CHECK-SAME: tensor<i32> + tosa.yield %4 : tensor<*xi32> + } + // CHECK: tosa.yield + // CHECK-SAME: tensor<i32> + tosa.yield %1 : tensor<*xi32> + } + + // CHECK: tensor.cast + return %1 : tensor<*xi32> +} + +// ----- + // CHECK-LABEL: @test_static_rfft2d func.func @test_static_rfft2d(%arg0: tensor<5x2x8xf32>) -> () { // CHECK: -> (tensor<5x2x5xf32>, tensor<5x2x5xf32>) diff --git a/mlir/test/Dialect/Vector/linearize.mlir b/mlir/test/Dialect/Vector/linearize.mlir index f0e9b3a..212541c 100644 --- a/mlir/test/Dialect/Vector/linearize.mlir +++ b/mlir/test/Dialect/Vector/linearize.mlir @@ -146,6 +146,16 @@ func.func @test_scalable_no_linearize(%arg0: vector<[2]x[2]xf32>) -> vector<[2]x // ----- +// ALL-LABEL: func.func @test_0d_vector +func.func @test_0d_vector() -> vector<f32> { + // ALL: %[[CST:.+]] = arith.constant dense<0.000000e+00> : vector<f32> + %0 = arith.constant dense<0.0> : vector<f32> + // ALL: return %[[CST]] + return %0 : vector<f32> +} + +// ----- + func.func @test_scalable_no_linearize(%arg0: vector<2x[2]xf32>) -> vector<2x[2]xf32> { // expected-error@+1 {{failed to legalize operation 'arith.constant' that was explicitly marked illegal}} %0 = arith.constant dense<[[1., 1.], [3., 3.]]> : vector<2x[2]xf32> diff --git a/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir b/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir index 3a120a5..252aeb0 100644 --- a/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir +++ b/mlir/test/Dialect/Vector/vector-dropleadunitdim-transforms.mlir @@ -238,6 +238,17 @@ func.func @cast_away_contraction_leading_one_dims_nonleadingunitdim_rank4_acctra return %0: vector<1x1x2x16xf32> } +// ----- + +// CHECK-LABEL: func.func @cast_away_contraction_does_not_transpose_leading_unit_dims +// CHECK-NOT vector.transpose +// CHECK: vector.contract +func.func @cast_away_contraction_does_not_transpose_leading_unit_dims(%lhs: vector<1x1x8xi32>, + %rhs: vector<1x8x8xi32>, + %acc: vector<1x8xi32>) -> vector<1x8xi32> { + %result = vector.contract {indexing_maps = [affine_map<(d0, d1, d2, d3) -> (d0, d1, d3)>, affine_map<(d0, d1, d2, d3) -> (d0, d2, d3)>, affine_map<(d0, d1, d2, d3) -> (d1, d2)>], iterator_types = ["parallel", "parallel", "parallel", "reduction"], kind = #vector.kind<add>} %lhs, %rhs, %acc : vector<1x1x8xi32>, vector<1x8x8xi32> into vector<1x8xi32> + return %result : vector<1x8xi32> +} // ----- // CHECK-LABEL: func @cast_away_extract_strided_slice_leading_one_dims @@ -663,4 +674,3 @@ func.func @drop_unit_dims_scalar_cond_select(%cond: i1, %arg0: vector<1x16xi1>, %sel = arith.select %cond, %arg0, %arg1 : vector<1x16xi1> return %sel : vector<1x16xi1> } - diff --git a/mlir/test/Pass/pass-timing.mlir b/mlir/test/Pass/pass-timing.mlir index bd5d611..cfb4b74 100644 --- a/mlir/test/Pass/pass-timing.mlir +++ b/mlir/test/Pass/pass-timing.mlir @@ -1,5 +1,7 @@ // RUN: mlir-opt %s -mlir-disable-threading=true -verify-each=true -pass-pipeline='builtin.module(func.func(cse,canonicalize,cse))' -mlir-timing -mlir-timing-display=list 2>&1 | FileCheck -check-prefix=LIST %s +// RUN: mlir-opt %s -mlir-disable-threading=true -verify-each=true -pass-pipeline='builtin.module(func.func(cse,canonicalize,cse))' -mlir-timing -mlir-timing-display=list -mlir-output-format=json 2>&1 | FileCheck -check-prefix=LIST-JSON %s // RUN: mlir-opt %s -mlir-disable-threading=true -verify-each=true -pass-pipeline='builtin.module(func.func(cse,canonicalize,cse))' -mlir-timing -mlir-timing-display=tree 2>&1 | FileCheck -check-prefix=PIPELINE %s +// RUN: mlir-opt %s -mlir-disable-threading=true -verify-each=true -pass-pipeline='builtin.module(func.func(cse,canonicalize,cse))' -mlir-timing -mlir-timing-display=tree -mlir-output-format=json 2>&1 | FileCheck -check-prefix=PIPELINE-JSON %s // RUN: mlir-opt %s -mlir-disable-threading=false -verify-each=true -pass-pipeline='builtin.module(func.func(cse,canonicalize,cse))' -mlir-timing -mlir-timing-display=list 2>&1 | FileCheck -check-prefix=MT_LIST %s // RUN: mlir-opt %s -mlir-disable-threading=false -verify-each=true -pass-pipeline='builtin.module(func.func(cse,canonicalize,cse))' -mlir-timing -mlir-timing-display=tree 2>&1 | FileCheck -check-prefix=MT_PIPELINE %s // RUN: mlir-opt %s -mlir-disable-threading=true -verify-each=false -test-pm-nested-pipeline -mlir-timing -mlir-timing-display=tree 2>&1 | FileCheck -check-prefix=NESTED_PIPELINE %s @@ -12,6 +14,14 @@ // LIST-DAG: DominanceInfo // LIST: Total +// LIST-JSON-NOT: Execution time report +// LIST-JSON-NOT: Total Execution Time: +// LIST-JSON-NOT: Name +// LIST-JSON-DAG: "name": "Canonicalizer"} +// LIST-JSON-DAG: "name": "CSE"} +// LIST-JSON-DAG: "name": "(A) DominanceInfo"} +// LIST-JSON: "name": "Total"} + // PIPELINE: Execution time report // PIPELINE: Total Execution Time: // PIPELINE: Name @@ -26,6 +36,28 @@ // PIPELINE-NEXT: Rest // PIPELINE-NEXT: Total +// PIPELINE-JSON-NOT: Execution time report +// PIPELINE-JSON-NOT: Total Execution Time: +// PIPELINE-JSON-NOT: Name +// PIPELINE-JSON: "name": "Parser", "passes": [ +// PIPELINE-JSON-NEXT: {}]}, +// PIPELINE-JSON-NEXT: "name": "'func.func' Pipeline", "passes": [ +// PIPELINE-JSON-NEXT: "name": "CSE", "passes": [ +// PIPELINE-JSON-NEXT: "name": "(A) DominanceInfo", "passes": [ +// PIPELINE-JSON-NEXT: {}]}, +// PIPELINE-JSON-NEXT: {}]}, +// PIPELINE-JSON-NEXT: "name": "Canonicalizer", "passes": [ +// PIPELINE-JSON-NEXT: {}]}, +// PIPELINE-JSON-NEXT: "name": "CSE", "passes": [ +// PIPELINE-JSON-NEXT: "name": "(A) DominanceInfo", "passes": [ +// PIPELINE-JSON-NEXT: {}]}, +// PIPELINE-JSON-NEXT: {}]}, +// PIPELINE-JSON-NEXT: {}]}, +// PIPELINE-JSON-NEXT: "name": "Output", "passes": [ +// PIPELINE-JSON-NEXT: {}]}, +// PIPELINE-JSON-NEXT: "name": "Rest" +// PIPELINE-JSON-NEXT: "name": "Total" + // MT_LIST: Execution time report // MT_LIST: Total Execution Time: // MT_LIST: Name diff --git a/mlir/test/Target/Cpp/subscript.mlir b/mlir/test/Target/Cpp/subscript.mlir index a6c82df..0b38895 100644 --- a/mlir/test/Target/Cpp/subscript.mlir +++ b/mlir/test/Target/Cpp/subscript.mlir @@ -1,24 +1,44 @@ // RUN: mlir-translate -mlir-to-cpp %s | FileCheck %s // RUN: mlir-translate -mlir-to-cpp -declare-variables-at-top %s | FileCheck %s -func.func @load_store(%arg0: !emitc.array<4x8xf32>, %arg1: !emitc.array<3x5xf32>, %arg2: index, %arg3: index) { - %0 = emitc.subscript %arg0[%arg2, %arg3] : <4x8xf32>, index, index - %1 = emitc.subscript %arg1[%arg2, %arg3] : <3x5xf32>, index, index +func.func @load_store_array(%arg0: !emitc.array<4x8xf32>, %arg1: !emitc.array<3x5xf32>, %arg2: index, %arg3: index) { + %0 = emitc.subscript %arg0[%arg2, %arg3] : (!emitc.array<4x8xf32>, index, index) -> f32 + %1 = emitc.subscript %arg1[%arg2, %arg3] : (!emitc.array<3x5xf32>, index, index) -> f32 emitc.assign %0 : f32 to %1 : f32 return } -// CHECK: void load_store(float [[ARR1:[^ ]*]][4][8], float [[ARR2:[^ ]*]][3][5], +// CHECK: void load_store_array(float [[ARR1:[^ ]*]][4][8], float [[ARR2:[^ ]*]][3][5], // CHECK-SAME: size_t [[I:[^ ]*]], size_t [[J:[^ ]*]]) // CHECK-NEXT: [[ARR2]][[[I]]][[[J]]] = [[ARR1]][[[I]]][[[J]]]; +func.func @load_store_pointer(%arg0: !emitc.ptr<f32>, %arg1: !emitc.ptr<f32>, %arg2: index, %arg3: index) { + %0 = emitc.subscript %arg0[%arg2] : (!emitc.ptr<f32>, index) -> f32 + %1 = emitc.subscript %arg1[%arg3] : (!emitc.ptr<f32>, index) -> f32 + emitc.assign %0 : f32 to %1 : f32 + return +} +// CHECK: void load_store_pointer(float* [[PTR1:[^ ]*]], float* [[PTR2:[^ ]*]], +// CHECK-SAME: size_t [[I:[^ ]*]], size_t [[J:[^ ]*]]) +// CHECK-NEXT: [[PTR2]][[[J]]] = [[PTR1]][[[I]]]; + +func.func @load_store_opaque(%arg0: !emitc.opaque<"std::map<char, int>">, %arg1: !emitc.opaque<"std::map<char, int>">, %arg2: !emitc.opaque<"char">, %arg3: !emitc.opaque<"char">) { + %0 = emitc.subscript %arg0[%arg2] : (!emitc.opaque<"std::map<char, int>">, !emitc.opaque<"char">) -> !emitc.opaque<"int"> + %1 = emitc.subscript %arg1[%arg3] : (!emitc.opaque<"std::map<char, int>">, !emitc.opaque<"char">) -> !emitc.opaque<"int"> + emitc.assign %0 : !emitc.opaque<"int"> to %1 : !emitc.opaque<"int"> + return +} +// CHECK: void load_store_opaque(std::map<char, int> [[MAP1:[^ ]*]], std::map<char, int> [[MAP2:[^ ]*]], +// CHECK-SAME: char [[I:[^ ]*]], char [[J:[^ ]*]]) +// CHECK-NEXT: [[MAP2]][[[J]]] = [[MAP1]][[[I]]]; + emitc.func @func1(%arg0 : f32) { emitc.return } emitc.func @call_arg(%arg0: !emitc.array<4x8xf32>, %i: i32, %j: i16, %k: i8) { - %0 = emitc.subscript %arg0[%i, %j] : <4x8xf32>, i32, i16 - %1 = emitc.subscript %arg0[%j, %k] : <4x8xf32>, i16, i8 + %0 = emitc.subscript %arg0[%i, %j] : (!emitc.array<4x8xf32>, i32, i16) -> f32 + %1 = emitc.subscript %arg0[%j, %k] : (!emitc.array<4x8xf32>, i16, i8) -> f32 emitc.call @func1 (%0) : (f32) -> () emitc.call_opaque "func2" (%1) : (f32) -> () diff --git a/mlir/test/Target/LLVMIR/openmp-parallel-reduction-cleanup.mlir b/mlir/test/Target/LLVMIR/openmp-parallel-reduction-cleanup.mlir new file mode 100644 index 0000000..9ae4c4a --- /dev/null +++ b/mlir/test/Target/LLVMIR/openmp-parallel-reduction-cleanup.mlir @@ -0,0 +1,94 @@ +// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s + +// test a parallel reduction with a cleanup region + + omp.declare_reduction @add_reduction_i_32 : !llvm.ptr init { + ^bb0(%arg0: !llvm.ptr): + %0 = llvm.mlir.constant(0 : i32) : i32 + %c4 = llvm.mlir.constant(4 : i64) : i64 + %2 = llvm.call @malloc(%c4) : (i64) -> !llvm.ptr + llvm.store %0, %2 : i32, !llvm.ptr + omp.yield(%2 : !llvm.ptr) + } combiner { + ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr): + %0 = llvm.load %arg0 : !llvm.ptr -> i32 + %1 = llvm.load %arg1 : !llvm.ptr -> i32 + %2 = llvm.add %0, %1 : i32 + llvm.store %2, %arg0 : i32, !llvm.ptr + omp.yield(%arg0 : !llvm.ptr) + } cleanup { + ^bb0(%arg0: !llvm.ptr): + llvm.call @free(%arg0) : (!llvm.ptr) -> () + omp.yield + } + + // CHECK-LABEL: @main + llvm.func @main() { + %0 = llvm.mlir.constant(-1 : i32) : i32 + %1 = llvm.mlir.addressof @i : !llvm.ptr + %2 = llvm.mlir.addressof @j : !llvm.ptr + omp.parallel byref reduction(@add_reduction_i_32 %1 -> %arg0 : !llvm.ptr, @add_reduction_i_32 %2 -> %arg1 : !llvm.ptr) { + llvm.store %0, %arg0 : i32, !llvm.ptr + llvm.store %0, %arg1 : i32, !llvm.ptr + omp.terminator + } + llvm.return + } + llvm.mlir.global internal @i() {addr_space = 0 : i32} : i32 { + %0 = llvm.mlir.constant(0 : i32) : i32 + llvm.return %0 : i32 + } + llvm.mlir.global internal @j() {addr_space = 0 : i32} : i32 { + %0 = llvm.mlir.constant(0 : i32) : i32 + llvm.return %0 : i32 + } + llvm.func @malloc(%arg0 : i64) -> !llvm.ptr + llvm.func @free(%arg0 : !llvm.ptr) -> () + +// CHECK: %{{.+}} = +// Call to the outlined function. +// CHECK: call void {{.*}} @__kmpc_fork_call +// CHECK-SAME: @[[OUTLINED:[A-Za-z_.][A-Za-z0-9_.]*]] + +// Outlined function. +// CHECK: define internal void @[[OUTLINED]] + +// Private reduction variable and its initialization. +// CHECK: %tid.addr.local = alloca i32 +// CHECK: %[[MALLOC_I:.+]] = call ptr @malloc(i64 4) +// CHECK: %[[PRIV_PTR_I:.+]] = alloca ptr +// CHECK: store ptr %[[MALLOC_I]], ptr %[[PRIV_PTR_I]] +// CHECK: %[[MALLOC_J:.+]] = call ptr @malloc(i64 4) +// CHECK: %[[PRIV_PTR_J:.+]] = alloca ptr +// CHECK: store ptr %[[MALLOC_J]], ptr %[[PRIV_PTR_J]] + +// Call to the reduction function. +// CHECK: call i32 @__kmpc_reduce +// CHECK-SAME: @[[REDFUNC:[A-Za-z_.][A-Za-z0-9_.]*]] + + +// Non-atomic reduction: +// CHECK: %[[PRIV_VAL_PTR_I:.+]] = load ptr, ptr %[[PRIV_PTR_I]] +// CHECK: %[[LOAD_I:.+]] = load i32, ptr @i +// CHECK: %[[PRIV_VAL_I:.+]] = load i32, ptr %[[PRIV_VAL_PTR_I]] +// CHECK: %[[SUM_I:.+]] = add i32 %[[LOAD_I]], %[[PRIV_VAL_I]] +// CHECK: store i32 %[[SUM_I]], ptr @i +// CHECK: %[[PRIV_VAL_PTR_J:.+]] = load ptr, ptr %[[PRIV_PTR_J]] +// CHECK: %[[LOAD_J:.+]] = load i32, ptr @j +// CHECK: %[[PRIV_VAL_J:.+]] = load i32, ptr %[[PRIV_VAL_PTR_J]] +// CHECK: %[[SUM_J:.+]] = add i32 %[[LOAD_J]], %[[PRIV_VAL_J]] +// CHECK: store i32 %[[SUM_J]], ptr @j +// CHECK: call void @__kmpc_end_reduce +// CHECK: br label %[[FINALIZE:.+]] + +// CHECK: [[FINALIZE]]: +// CHECK: br label %[[OMP_FINALIZE:.+]] + +// Cleanup region: +// CHECK: [[OMP_FINALIZE]]: +// CHECK: call void @free(ptr %[[PRIV_PTR_I]]) +// CHECK: call void @free(ptr %[[PRIV_PTR_J]]) + +// Reduction function. +// CHECK: define internal void @[[REDFUNC]] +// CHECK: add i32 diff --git a/mlir/test/Target/LLVMIR/openmp-reduction-init-arg.mlir b/mlir/test/Target/LLVMIR/openmp-reduction-init-arg.mlir new file mode 100644 index 0000000..5dd31c4 --- /dev/null +++ b/mlir/test/Target/LLVMIR/openmp-reduction-init-arg.mlir @@ -0,0 +1,111 @@ +// RUN: mlir-translate -mlir-to-llvmir %s | FileCheck %s + +// Test that the block argument to the initialization region of +// omp.declare_reduction gets mapped properly when translating to LLVMIR. + +module { + omp.declare_reduction @add_reduction_byref_box_Uxf64 : !llvm.ptr init { + ^bb0(%arg0: !llvm.ptr): +// test usage of %arg0: + %11 = llvm.load %arg0 : !llvm.ptr -> !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> + omp.yield(%arg0 : !llvm.ptr) + } combiner { + ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr): + omp.yield(%arg0 : !llvm.ptr) + } + + llvm.func internal @_QFPreduce(%arg0: !llvm.ptr {fir.bindc_name = "r"}, %arg1: !llvm.ptr {fir.bindc_name = "r2"}) attributes {sym_visibility = "private"} { + %8 = llvm.mlir.constant(1 : i32) : i32 + %9 = llvm.mlir.constant(10 : i32) : i32 + %10 = llvm.mlir.constant(0 : i32) : i32 + %83 = llvm.mlir.constant(1 : i64) : i64 + %84 = llvm.alloca %83 x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> : (i64) -> !llvm.ptr + %86 = llvm.mlir.constant(1 : i64) : i64 + %87 = llvm.alloca %86 x !llvm.struct<(ptr, i64, i32, i8, i8, i8, i8, array<1 x array<3 x i64>>)> : (i64) -> !llvm.ptr +// test multiple reduction variables to ensure they don't intefere with eachother +// when inlining the reduction init region multiple times + omp.parallel byref reduction(@add_reduction_byref_box_Uxf64 %84 -> %arg3 : !llvm.ptr, @add_reduction_byref_box_Uxf64 %87 -> %arg4 : !llvm.ptr) { + omp.terminator + } + llvm.return + } +} + +// CHECK-LABEL: define internal void @_QFPreduce +// CHECK: %[[VAL_0:.*]] = alloca { ptr, ptr }, align 8 +// CHECK: %[[VAL_1:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, i64 1, align 8 +// CHECK: %[[VAL_2:.*]] = alloca { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, i64 1, align 8 +// CHECK: br label %[[VAL_3:.*]] +// CHECK: entry: ; preds = %[[VAL_4:.*]] +// CHECK: %[[VAL_5:.*]] = call i32 @__kmpc_global_thread_num(ptr @1) +// CHECK: br label %[[VAL_6:.*]] +// CHECK: omp_parallel: ; preds = %[[VAL_3]] +// CHECK: %[[VAL_7:.*]] = getelementptr { ptr, ptr }, ptr %[[VAL_0]], i32 0, i32 0 +// CHECK: store ptr %[[VAL_1]], ptr %[[VAL_7]], align 8 +// CHECK: %[[VAL_8:.*]] = getelementptr { ptr, ptr }, ptr %[[VAL_0]], i32 0, i32 1 +// CHECK: store ptr %[[VAL_2]], ptr %[[VAL_8]], align 8 +// CHECK: call void (ptr, i32, ptr, ...) @__kmpc_fork_call(ptr @1, i32 1, ptr @_QFPreduce..omp_par, ptr %[[VAL_0]]) +// CHECK: br label %[[VAL_9:.*]] +// CHECK: omp.par.outlined.exit: ; preds = %[[VAL_6]] +// CHECK: br label %[[VAL_10:.*]] +// CHECK: omp.par.exit.split: ; preds = %[[VAL_9]] +// CHECK: ret void +// CHECK: omp.par.entry: +// CHECK: %[[VAL_11:.*]] = getelementptr { ptr, ptr }, ptr %[[VAL_12:.*]], i32 0, i32 0 +// CHECK: %[[VAL_13:.*]] = load ptr, ptr %[[VAL_11]], align 8 +// CHECK: %[[VAL_14:.*]] = getelementptr { ptr, ptr }, ptr %[[VAL_12]], i32 0, i32 1 +// CHECK: %[[VAL_15:.*]] = load ptr, ptr %[[VAL_14]], align 8 +// CHECK: %[[VAL_16:.*]] = alloca i32, align 4 +// CHECK: %[[VAL_17:.*]] = load i32, ptr %[[VAL_18:.*]], align 4 +// CHECK: store i32 %[[VAL_17]], ptr %[[VAL_16]], align 4 +// CHECK: %[[VAL_19:.*]] = load i32, ptr %[[VAL_16]], align 4 +// CHECK: %[[VAL_20:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[VAL_13]], align 8 +// CHECK: %[[VAL_21:.*]] = alloca ptr, align 8 +// CHECK: store ptr %[[VAL_13]], ptr %[[VAL_21]], align 8 +// CHECK: %[[VAL_22:.*]] = load { ptr, i64, i32, i8, i8, i8, i8, [1 x [3 x i64]] }, ptr %[[VAL_15]], align 8 +// CHECK: %[[VAL_23:.*]] = alloca ptr, align 8 +// CHECK: store ptr %[[VAL_15]], ptr %[[VAL_23]], align 8 +// CHECK: %[[VAL_24:.*]] = alloca [2 x ptr], align 8 +// CHECK: br label %[[VAL_25:.*]] +// CHECK: omp.par.region: ; preds = %[[VAL_26:.*]] +// CHECK: br label %[[VAL_27:.*]] +// CHECK: omp.par.region1: ; preds = %[[VAL_25]] +// CHECK: br label %[[VAL_28:.*]] +// CHECK: omp.region.cont: ; preds = %[[VAL_27]] +// CHECK: %[[VAL_29:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_24]], i64 0, i64 0 +// CHECK: store ptr %[[VAL_21]], ptr %[[VAL_29]], align 8 +// CHECK: %[[VAL_30:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_24]], i64 0, i64 1 +// CHECK: store ptr %[[VAL_23]], ptr %[[VAL_30]], align 8 +// CHECK: %[[VAL_31:.*]] = call i32 @__kmpc_global_thread_num(ptr @1) +// CHECK: %[[VAL_32:.*]] = call i32 @__kmpc_reduce(ptr @1, i32 %[[VAL_31]], i32 2, i64 16, ptr %[[VAL_24]], ptr @.omp.reduction.func, ptr @.gomp_critical_user_.reduction.var) +// CHECK: switch i32 %[[VAL_32]], label %[[VAL_33:.*]] [ +// CHECK: i32 1, label %[[VAL_34:.*]] +// CHECK: i32 2, label %[[VAL_35:.*]] +// CHECK: ] +// CHECK: reduce.switch.atomic: ; preds = %[[VAL_28]] +// CHECK: unreachable +// CHECK: reduce.switch.nonatomic: ; preds = %[[VAL_28]] +// CHECK: %[[VAL_36:.*]] = load ptr, ptr %[[VAL_21]], align 8 +// CHECK: %[[VAL_37:.*]] = load ptr, ptr %[[VAL_23]], align 8 +// CHECK: call void @__kmpc_end_reduce(ptr @1, i32 %[[VAL_31]], ptr @.gomp_critical_user_.reduction.var) +// CHECK: br label %[[VAL_33]] +// CHECK: reduce.finalize: ; preds = %[[VAL_34]], %[[VAL_28]] +// CHECK: br label %[[VAL_38:.*]] +// CHECK: omp.par.pre_finalize: ; preds = %[[VAL_33]] +// CHECK: br label %[[VAL_39:.*]] +// CHECK: omp.par.outlined.exit.exitStub: ; preds = %[[VAL_38]] +// CHECK: ret void +// CHECK: %[[VAL_40:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_41:.*]], i64 0, i64 0 +// CHECK: %[[VAL_42:.*]] = load ptr, ptr %[[VAL_40]], align 8 +// CHECK: %[[VAL_43:.*]] = load ptr, ptr %[[VAL_42]], align 8 +// CHECK: %[[VAL_44:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_45:.*]], i64 0, i64 0 +// CHECK: %[[VAL_46:.*]] = load ptr, ptr %[[VAL_44]], align 8 +// CHECK: %[[VAL_47:.*]] = load ptr, ptr %[[VAL_46]], align 8 +// CHECK: %[[VAL_48:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_41]], i64 0, i64 1 +// CHECK: %[[VAL_49:.*]] = load ptr, ptr %[[VAL_48]], align 8 +// CHECK: %[[VAL_50:.*]] = load ptr, ptr %[[VAL_49]], align 8 +// CHECK: %[[VAL_51:.*]] = getelementptr inbounds [2 x ptr], ptr %[[VAL_45]], i64 0, i64 1 +// CHECK: %[[VAL_52:.*]] = load ptr, ptr %[[VAL_51]], align 8 +// CHECK: %[[VAL_53:.*]] = load ptr, ptr %[[VAL_52]], align 8 +// CHECK: ret void + diff --git a/mlir/test/Target/LLVMIR/openmp-wsloop-reduction-cleanup.mlir b/mlir/test/Target/LLVMIR/openmp-wsloop-reduction-cleanup.mlir new file mode 100644 index 0000000..a1e17af --- /dev/null +++ b/mlir/test/Target/LLVMIR/openmp-wsloop-reduction-cleanup.mlir @@ -0,0 +1,86 @@ +// RUN: mlir-translate -mlir-to-llvmir -split-input-file %s | FileCheck %s + +// test a wsloop reduction with a cleanup region + + omp.declare_reduction @add_reduction_i_32 : !llvm.ptr init { + ^bb0(%arg0: !llvm.ptr): + %0 = llvm.mlir.constant(0 : i32) : i32 + %c4 = llvm.mlir.constant(4 : i64) : i64 + %2 = llvm.call @malloc(%c4) : (i64) -> !llvm.ptr + llvm.store %0, %2 : i32, !llvm.ptr + omp.yield(%2 : !llvm.ptr) + } combiner { + ^bb0(%arg0: !llvm.ptr, %arg1: !llvm.ptr): + %0 = llvm.load %arg0 : !llvm.ptr -> i32 + %1 = llvm.load %arg1 : !llvm.ptr -> i32 + %2 = llvm.add %0, %1 : i32 + llvm.store %2, %arg0 : i32, !llvm.ptr + omp.yield(%arg0 : !llvm.ptr) + } cleanup { + ^bb0(%arg0: !llvm.ptr): + llvm.call @free(%arg0) : (!llvm.ptr) -> () + omp.yield + } + + // CHECK-LABEL: @main + llvm.func @main() { + %0 = llvm.mlir.constant(-1 : i32) : i32 + %1 = llvm.mlir.addressof @i : !llvm.ptr + %2 = llvm.mlir.addressof @j : !llvm.ptr + %loop_ub = llvm.mlir.constant(9 : i32) : i32 + %loop_lb = llvm.mlir.constant(0 : i32) : i32 + %loop_step = llvm.mlir.constant(1 : i32) : i32 + omp.wsloop byref reduction(@add_reduction_i_32 %1 -> %arg0 : !llvm.ptr, @add_reduction_i_32 %2 -> %arg1 : !llvm.ptr) for (%loop_cnt) : i32 = (%loop_lb) to (%loop_ub) inclusive step (%loop_step) { + llvm.store %0, %arg0 : i32, !llvm.ptr + llvm.store %0, %arg1 : i32, !llvm.ptr + omp.terminator + } + llvm.return + } + llvm.mlir.global internal @i() {addr_space = 0 : i32} : i32 { + %0 = llvm.mlir.constant(0 : i32) : i32 + llvm.return %0 : i32 + } + llvm.mlir.global internal @j() {addr_space = 0 : i32} : i32 { + %0 = llvm.mlir.constant(0 : i32) : i32 + llvm.return %0 : i32 + } + llvm.func @malloc(%arg0 : i64) -> !llvm.ptr + llvm.func @free(%arg0 : !llvm.ptr) -> () + +// Private reduction variable and its initialization. +// CHECK: %[[MALLOC_I:.+]] = call ptr @malloc(i64 4) +// CHECK: %[[PRIV_PTR_I:.+]] = alloca ptr +// CHECK: store ptr %[[MALLOC_I]], ptr %[[PRIV_PTR_I]] +// CHECK: %[[MALLOC_J:.+]] = call ptr @malloc(i64 4) +// CHECK: %[[PRIV_PTR_J:.+]] = alloca ptr +// CHECK: store ptr %[[MALLOC_J]], ptr %[[PRIV_PTR_J]] + +// Call to the reduction function. +// CHECK: call i32 @__kmpc_reduce +// CHECK-SAME: @[[REDFUNC:[A-Za-z_.][A-Za-z0-9_.]*]] + +// Weirdly the finalization block is generated before the reduction blocks: +// CHECK: [[FINALIZE:.+]]: +// CHECK: call void @__kmpc_barrier +// CHECK: call void @free(ptr %[[PRIV_PTR_I]]) +// CHECK: call void @free(ptr %[[PRIV_PTR_J]]) +// CHECK: ret void + +// Non-atomic reduction: +// CHECK: %[[PRIV_VAL_PTR_I:.+]] = load ptr, ptr %[[PRIV_PTR_I]] +// CHECK: %[[LOAD_I:.+]] = load i32, ptr @i +// CHECK: %[[PRIV_VAL_I:.+]] = load i32, ptr %[[PRIV_VAL_PTR_I]] +// CHECK: %[[SUM_I:.+]] = add i32 %[[LOAD_I]], %[[PRIV_VAL_I]] +// CHECK: store i32 %[[SUM_I]], ptr @i +// CHECK: %[[PRIV_VAL_PTR_J:.+]] = load ptr, ptr %[[PRIV_PTR_J]] +// CHECK: %[[LOAD_J:.+]] = load i32, ptr @j +// CHECK: %[[PRIV_VAL_J:.+]] = load i32, ptr %[[PRIV_VAL_PTR_J]] +// CHECK: %[[SUM_J:.+]] = add i32 %[[LOAD_J]], %[[PRIV_VAL_J]] +// CHECK: store i32 %[[SUM_J]], ptr @j +// CHECK: call void @__kmpc_end_reduce +// CHECK: br label %[[FINALIZE]] + +// Reduction function. +// CHECK: define internal void @[[REDFUNC]] +// CHECK: add i32 diff --git a/mlir/test/Transforms/parallel-loop-collapsing.mlir b/mlir/test/Transforms/parallel-loop-collapsing.mlir index 660d7edb..d1c23d5 100644 --- a/mlir/test/Transforms/parallel-loop-collapsing.mlir +++ b/mlir/test/Transforms/parallel-loop-collapsing.mlir @@ -1,6 +1,6 @@ // RUN: mlir-opt -allow-unregistered-dialect %s -pass-pipeline='builtin.module(func.func(test-scf-parallel-loop-collapsing{collapsed-indices-0=0,3 collapsed-indices-1=1,4 collapsed-indices-2=2}, canonicalize))' | FileCheck %s -// CHECK-LABEL: func @parallel_many_dims() { +// CHECK: func @parallel_many_dims() { func.func @parallel_many_dims() { %c0 = arith.constant 0 : index %c1 = arith.constant 1 : index @@ -28,19 +28,19 @@ func.func @parallel_many_dims() { return } -// CHECK-DAG: [[C12:%.*]] = arith.constant 12 : index -// CHECK-DAG: [[C10:%.*]] = arith.constant 10 : index -// CHECK-DAG: [[C9:%.*]] = arith.constant 9 : index -// CHECK-DAG: [[C6:%.*]] = arith.constant 6 : index -// CHECK-DAG: [[C4:%.*]] = arith.constant 4 : index -// CHECK-DAG: [[C3:%.*]] = arith.constant 3 : index -// CHECK-DAG: [[C2:%.*]] = arith.constant 2 : index -// CHECK-DAG: [[C1:%.*]] = arith.constant 1 : index -// CHECK-DAG: [[C0:%.*]] = arith.constant 0 : index -// CHECK: scf.parallel ([[NEW_I0:%.*]]) = ([[C0]]) to ([[C4]]) step ([[C1]]) { -// CHECK: [[V0:%.*]] = arith.remsi [[NEW_I0]], [[C2]] : index -// CHECK: [[I0:%.*]] = arith.divsi [[NEW_I0]], [[C2]] : index -// CHECK: [[V2:%.*]] = arith.muli [[V0]], [[C10]] : index -// CHECK: [[I3:%.*]] = arith.addi [[V2]], [[C9]] : index -// CHECK: "magic.op"([[I0]], [[C3]], [[C6]], [[I3]], [[C12]]) : (index, index, index, index, index) -> index +// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index +// CHECK-DAG: %[[C6:.*]] = arith.constant 6 : index +// CHECK-DAG: %[[C12:.*]] = arith.constant 12 : index +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[C9:.*]] = arith.constant 9 : index +// CHECK-DAG: %[[C10:.*]] = arith.constant 10 : index +// CHECK-DAG: %[[C2:.*]] = arith.constant 2 : index +// CHECK-DAG: %[[C4:.*]] = arith.constant 4 : index +// CHECK: scf.parallel (%[[NEW_I0:.*]]) = (%[[C0]]) to (%[[C4]]) step (%[[C1]]) { +// CHECK: %[[V0:.*]] = arith.remsi %[[NEW_I0]], %[[C2]] : index +// CHECK: %[[I0:.*]] = arith.divsi %[[NEW_I0]], %[[C2]] : index +// CHECK: %[[V2:.*]] = arith.muli %[[V0]], %[[C10]] +// CHECK: %[[I3:.*]] = arith.addi %[[V2]], %[[C9]] +// CHECK: "magic.op"(%[[I0]], %[[C3]], %[[C6]], %[[I3]], %[[C12]]) : (index, index, index, index, index) -> index // CHECK: scf.reduce diff --git a/mlir/test/Transforms/single-parallel-loop-collapsing.mlir b/mlir/test/Transforms/single-parallel-loop-collapsing.mlir index 542786b..4eed61a 100644 --- a/mlir/test/Transforms/single-parallel-loop-collapsing.mlir +++ b/mlir/test/Transforms/single-parallel-loop-collapsing.mlir @@ -13,22 +13,22 @@ func.func @collapse_to_single() { return } -// CHECK-LABEL: func @collapse_to_single() { -// CHECK-DAG: [[C18:%.*]] = arith.constant 18 : index -// CHECK-DAG: [[C6:%.*]] = arith.constant 6 : index -// CHECK-DAG: [[C3:%.*]] = arith.constant 3 : index -// CHECK-DAG: [[C7:%.*]] = arith.constant 7 : index -// CHECK-DAG: [[C4:%.*]] = arith.constant 4 : index -// CHECK-DAG: [[C1:%.*]] = arith.constant 1 : index -// CHECK-DAG: [[C0:%.*]] = arith.constant 0 : index -// CHECK: scf.parallel ([[NEW_I:%.*]]) = ([[C0]]) to ([[C18]]) step ([[C1]]) { -// CHECK: [[I0_COUNT:%.*]] = arith.remsi [[NEW_I]], [[C6]] : index -// CHECK: [[I1_COUNT:%.*]] = arith.divsi [[NEW_I]], [[C6]] : index -// CHECK: [[V0:%.*]] = arith.muli [[I0_COUNT]], [[C4]] : index -// CHECK: [[I1:%.*]] = arith.addi [[V0]], [[C7]] : index -// CHECK: [[V1:%.*]] = arith.muli [[I1_COUNT]], [[C3]] : index -// CHECK: [[I0:%.*]] = arith.addi [[V1]], [[C3]] : index -// CHECK: "magic.op"([[I0]], [[I1]]) : (index, index) -> index +// CHECK: func @collapse_to_single() { +// CHECK-DAG: %[[C1:.*]] = arith.constant 1 : index +// CHECK-DAG: %[[C0:.*]] = arith.constant 0 : index +// CHECK-DAG: %[[C3:.*]] = arith.constant 3 : index +// CHECK-DAG: %[[C7:.*]] = arith.constant 7 : index +// CHECK-DAG: %[[C4:.*]] = arith.constant 4 : index +// CHECK-DAG: %[[C6:.*]] = arith.constant 6 : index +// CHECK-DAG: %[[C18:.*]] = arith.constant 18 : index +// CHECK: scf.parallel (%[[NEW_I:.*]]) = (%[[C0]]) to (%[[C18]]) step (%[[C1]]) { +// CHECK: %[[I0_COUNT:.*]] = arith.remsi %[[NEW_I]], %[[C6]] : index +// CHECK: %[[I1_COUNT:.*]] = arith.divsi %[[NEW_I]], %[[C6]] : index +// CHECK: %[[V0:.*]] = arith.muli %[[I0_COUNT]], %[[C4]] +// CHECK: %[[I1:.*]] = arith.addi %[[V0]], %[[C7]] +// CHECK: %[[V1:.*]] = arith.muli %[[I1_COUNT]], %[[C3]] +// CHECK: %[[I0:.*]] = arith.addi %[[V1]], %[[C3]] +// CHECK: "magic.op"(%[[I0]], %[[I1]]) : (index, index) -> index // CHECK: scf.reduce // CHECK-NEXT: } // CHECK-NEXT: return diff --git a/mlir/test/lib/Dialect/Affine/TestLoopFusion.cpp b/mlir/test/lib/Dialect/Affine/TestLoopFusion.cpp index f4f1593..1901180 100644 --- a/mlir/test/lib/Dialect/Affine/TestLoopFusion.cpp +++ b/mlir/test/lib/Dialect/Affine/TestLoopFusion.cpp @@ -22,23 +22,6 @@ using namespace mlir; using namespace mlir::affine; -static llvm::cl::OptionCategory clOptionsCategory(DEBUG_TYPE " options"); - -static llvm::cl::opt<bool> clTestDependenceCheck( - "test-loop-fusion-dependence-check", - llvm::cl::desc("Enable testing of loop fusion dependence check"), - llvm::cl::cat(clOptionsCategory)); - -static llvm::cl::opt<bool> clTestSliceComputation( - "test-loop-fusion-slice-computation", - llvm::cl::desc("Enable testing of loop fusion slice computation"), - llvm::cl::cat(clOptionsCategory)); - -static llvm::cl::opt<bool> clTestLoopFusionTransformation( - "test-loop-fusion-transformation", - llvm::cl::desc("Enable testing of loop fusion transformation"), - llvm::cl::cat(clOptionsCategory)); - namespace { struct TestLoopFusion @@ -50,6 +33,24 @@ struct TestLoopFusion return "Tests loop fusion utility functions."; } void runOnOperation() override; + + TestLoopFusion() = default; + TestLoopFusion(const TestLoopFusion &pass) : PassWrapper(pass){}; + + Option<bool> clTestDependenceCheck{ + *this, "test-loop-fusion-dependence-check", + llvm::cl::desc("Enable testing of loop fusion dependence check"), + llvm::cl::init(false)}; + + Option<bool> clTestSliceComputation{ + *this, "test-loop-fusion-slice-computation", + llvm::cl::desc("Enable testing of loop fusion slice computation"), + llvm::cl::init(false)}; + + Option<bool> clTestLoopFusionTransformation{ + *this, "test-loop-fusion-transformation", + llvm::cl::desc("Enable testing of loop fusion transformation"), + llvm::cl::init(false)}; }; } // namespace diff --git a/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp b/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp index 5e160b7..4b2b1a0 100644 --- a/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp +++ b/mlir/test/lib/Dialect/Affine/TestReifyValueBounds.cpp @@ -117,14 +117,17 @@ static LogicalResult testReifyValueBounds(func::FuncOp funcOp, // Prepare stop condition. By default, reify in terms of the op's // operands. No stop condition is used when a constant was requested. - std::function<bool(Value, std::optional<int64_t>)> stopCondition = - [&](Value v, std::optional<int64_t> d) { + std::function<bool(Value, std::optional<int64_t>, + ValueBoundsConstraintSet & cstr)> + stopCondition = [&](Value v, std::optional<int64_t> d, + ValueBoundsConstraintSet &cstr) { // Reify in terms of SSA values that are different from `value`. return v != value; }; if (reifyToFuncArgs) { // Reify in terms of function block arguments. - stopCondition = stopCondition = [](Value v, std::optional<int64_t> d) { + stopCondition = [](Value v, std::optional<int64_t> d, + ValueBoundsConstraintSet &cstr) { auto bbArg = dyn_cast<BlockArgument>(v); if (!bbArg) return false; diff --git a/mlir/test/lib/Dialect/Affine/TestVectorizationUtils.cpp b/mlir/test/lib/Dialect/Affine/TestVectorizationUtils.cpp index b497f8d..598678f 100644 --- a/mlir/test/lib/Dialect/Affine/TestVectorizationUtils.cpp +++ b/mlir/test/lib/Dialect/Affine/TestVectorizationUtils.cpp @@ -37,39 +37,6 @@ using namespace mlir::affine; static llvm::cl::OptionCategory clOptionsCategory(DEBUG_TYPE " options"); -static llvm::cl::list<int> clTestVectorShapeRatio( - "vector-shape-ratio", - llvm::cl::desc("Specify the HW vector size for vectorization"), - llvm::cl::cat(clOptionsCategory)); -static llvm::cl::opt<bool> clTestForwardSlicingAnalysis( - "forward-slicing", - llvm::cl::desc("Enable testing forward static slicing and topological sort " - "functionalities"), - llvm::cl::cat(clOptionsCategory)); -static llvm::cl::opt<bool> clTestBackwardSlicingAnalysis( - "backward-slicing", - llvm::cl::desc("Enable testing backward static slicing and " - "topological sort functionalities"), - llvm::cl::cat(clOptionsCategory)); -static llvm::cl::opt<bool> clTestSlicingAnalysis( - "slicing", - llvm::cl::desc("Enable testing static slicing and topological sort " - "functionalities"), - llvm::cl::cat(clOptionsCategory)); -static llvm::cl::opt<bool> clTestComposeMaps( - "compose-maps", - llvm::cl::desc( - "Enable testing the composition of AffineMap where each " - "AffineMap in the composition is specified as the affine_map attribute " - "in a constant op."), - llvm::cl::cat(clOptionsCategory)); -static llvm::cl::opt<bool> clTestVecAffineLoopNest( - "vectorize-affine-loop-nest", - llvm::cl::desc( - "Enable testing for the 'vectorizeAffineLoopNest' utility by " - "vectorizing the outermost loops found"), - llvm::cl::cat(clOptionsCategory)); - namespace { struct VectorizerTestPass : public PassWrapper<VectorizerTestPass, OperationPass<func::FuncOp>> { @@ -85,6 +52,37 @@ struct VectorizerTestPass return "Tests vectorizer standalone functionality."; } + VectorizerTestPass() = default; + VectorizerTestPass(const VectorizerTestPass &pass) : PassWrapper(pass){}; + + ListOption<int> clTestVectorShapeRatio{ + *this, "vector-shape-ratio", + llvm::cl::desc("Specify the HW vector size for vectorization")}; + Option<bool> clTestForwardSlicingAnalysis{ + *this, "forward-slicing", + llvm::cl::desc( + "Enable testing forward static slicing and topological sort " + "functionalities")}; + Option<bool> clTestBackwardSlicingAnalysis{ + *this, "backward-slicing", + llvm::cl::desc("Enable testing backward static slicing and " + "topological sort functionalities")}; + Option<bool> clTestSlicingAnalysis{ + *this, "slicing", + llvm::cl::desc("Enable testing static slicing and topological sort " + "functionalities")}; + Option<bool> clTestComposeMaps{ + *this, "compose-maps", + llvm::cl::desc("Enable testing the composition of AffineMap where each " + "AffineMap in the composition is specified as the " + "affine_map attribute " + "in a constant op.")}; + Option<bool> clTestVecAffineLoopNest{ + *this, "vectorize-affine-loop-nest", + llvm::cl::desc( + "Enable testing for the 'vectorizeAffineLoopNest' utility by " + "vectorizing the outermost loops found")}; + void runOnOperation() override; void testVectorShapeRatio(llvm::raw_ostream &outs); void testForwardSlicing(llvm::raw_ostream &outs); diff --git a/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp b/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp index 3da48ff..a4464bb 100644 --- a/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp +++ b/mlir/test/lib/Dialect/DLTI/TestDataLayoutQuery.cpp @@ -41,6 +41,7 @@ struct TestDataLayoutQuery uint64_t alignment = layout.getTypeABIAlignment(op.getType()); uint64_t preferred = layout.getTypePreferredAlignment(op.getType()); uint64_t index = layout.getTypeIndexBitwidth(op.getType()).value_or(0); + Attribute endianness = layout.getEndianness(); Attribute allocaMemorySpace = layout.getAllocaMemorySpace(); Attribute programMemorySpace = layout.getProgramMemorySpace(); Attribute globalMemorySpace = layout.getGlobalMemorySpace(); @@ -51,6 +52,9 @@ struct TestDataLayoutQuery builder.getNamedAttr("alignment", builder.getIndexAttr(alignment)), builder.getNamedAttr("preferred", builder.getIndexAttr(preferred)), builder.getNamedAttr("index", builder.getIndexAttr(index)), + builder.getNamedAttr("endianness", endianness == Attribute() + ? builder.getStringAttr("") + : endianness), builder.getNamedAttr("alloca_memory_space", allocaMemorySpace == Attribute() ? builder.getUI32IntegerAttr(0) diff --git a/mlir/test/mlir-tblgen/op-properties.td b/mlir/test/mlir-tblgen/op-properties.td new file mode 100644 index 0000000..a484f68 --- /dev/null +++ b/mlir/test/mlir-tblgen/op-properties.td @@ -0,0 +1,21 @@ +// RUN: mlir-tblgen -gen-op-defs -I %S/../../include %s | FileCheck %s + +include "mlir/IR/AttrTypeBase.td" +include "mlir/IR/EnumAttr.td" +include "mlir/IR/OpBase.td" + +def Test_Dialect : Dialect { + let name = "test"; + let cppNamespace = "foobar"; +} +class NS_Op<string mnemonic, list<Trait> traits = []> : + Op<Test_Dialect, mnemonic, traits>; + +def OpWithAttr : NS_Op<"op_with_attr">{ + let arguments = (ins AnyAttr:$attr, OptionalAttr<AnyAttr>:$optional); +} + +// CHECK: void OpWithAttr::setAttrAttr(::mlir::Attribute attr) +// CHECK-NEXT: getProperties().attr = attr +// CHECK: void OpWithAttr::setOptionalAttr(::mlir::Attribute attr) +// CHECK-NEXT: getProperties().optional = attr diff --git a/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp b/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp index f14e559..fe6ad15 100644 --- a/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp +++ b/mlir/tools/mlir-linalg-ods-gen/mlir-linalg-ods-yaml-gen.cpp @@ -1008,7 +1008,7 @@ void {0}::regionBuilder(ImplicitLocOpBuilder &b, Block &block, ArrayRef<NamedAttribute> attrs) {{ assert({1} > 0 && block.getNumArguments() == {1} && "{0} regionBuilder expects {1} (>=0) args"); - RegionBuilderHelper helper(block.getArgument(0).getContext(), block); + RegionBuilderHelper helper(b, block); SmallVector<Value> yields; {2} {3} diff --git a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp index 3a69752..843760d 100644 --- a/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp +++ b/mlir/tools/mlir-tblgen/OpDefinitionsGen.cpp @@ -1804,23 +1804,36 @@ void OpEmitter::genAttrGetters() { } void OpEmitter::genAttrSetters() { + bool useProperties = op.getDialect().usePropertiesForAttributes(); + + // Generate the code to set an attribute. + auto emitSetAttr = [&](Method *method, StringRef getterName, + StringRef attrName, StringRef attrVar) { + if (useProperties) { + method->body() << formatv(" getProperties().{0} = {1};", attrName, + attrVar); + } else { + method->body() << formatv(" (*this)->setAttr({0}AttrName(), {1});", + getterName, attrVar); + } + }; + // Generate raw named setter type. This is a wrapper class that allows setting // to the attributes via setters instead of having to use the string interface // for better compile time verification. auto emitAttrWithStorageType = [&](StringRef setterName, StringRef getterName, - Attribute attr) { + StringRef attrName, Attribute attr) { auto *method = opClass.addMethod("void", setterName + "Attr", MethodParameter(attr.getStorageType(), "attr")); if (method) - method->body() << formatv(" (*this)->setAttr({0}AttrName(), attr);", - getterName); + emitSetAttr(method, getterName, attrName, "attr"); }; // Generate a setter that accepts the underlying C++ type as opposed to the // attribute type. auto emitAttrWithReturnType = [&](StringRef setterName, StringRef getterName, - Attribute attr) { + StringRef attrName, Attribute attr) { Attribute baseAttr = attr.getBaseAttr(); if (!canUseUnwrappedRawValue(baseAttr)) return; @@ -1849,9 +1862,8 @@ void OpEmitter::genAttrSetters() { // If the value isn't optional, just set it directly. if (!isOptional) { - method->body() << formatv( - " (*this)->setAttr({0}AttrName(), {1});", getterName, - constBuildAttrFromParam(attr, fctx, "attrValue")); + emitSetAttr(method, getterName, attrName, + constBuildAttrFromParam(attr, fctx, "attrValue")); return; } @@ -1862,13 +1874,25 @@ void OpEmitter::genAttrSetters() { // optional but not in the same way as the others (i.e. it uses bool over // std::optional<>). StringRef paramStr = isUnitAttr ? "attrValue" : "*attrValue"; - const char *optionalCodeBody = R"( + if (!useProperties) { + const char *optionalCodeBody = R"( if (attrValue) return (*this)->setAttr({0}AttrName(), {1}); (*this)->removeAttr({0}AttrName());)"; - method->body() << formatv( - optionalCodeBody, getterName, - constBuildAttrFromParam(baseAttr, fctx, paramStr)); + method->body() << formatv( + optionalCodeBody, getterName, + constBuildAttrFromParam(baseAttr, fctx, paramStr)); + } else { + const char *optionalCodeBody = R"( + auto &odsProp = getProperties().{0}; + if (attrValue) + odsProp = {1}; + else + odsProp = nullptr;)"; + method->body() << formatv( + optionalCodeBody, attrName, + constBuildAttrFromParam(baseAttr, fctx, paramStr)); + } }; for (const NamedAttribute &namedAttr : op.getAttributes()) { @@ -1876,8 +1900,10 @@ void OpEmitter::genAttrSetters() { continue; std::string setterName = op.getSetterName(namedAttr.name); std::string getterName = op.getGetterName(namedAttr.name); - emitAttrWithStorageType(setterName, getterName, namedAttr.attr); - emitAttrWithReturnType(setterName, getterName, namedAttr.attr); + emitAttrWithStorageType(setterName, getterName, namedAttr.name, + namedAttr.attr); + emitAttrWithReturnType(setterName, getterName, namedAttr.name, + namedAttr.attr); } } diff --git a/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp b/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp index d6b8d73..5f48429 100644 --- a/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp +++ b/mlir/unittests/Interfaces/DataLayoutInterfacesTest.cpp @@ -22,6 +22,7 @@ using namespace mlir; namespace { constexpr static llvm::StringLiteral kAttrName = "dltest.layout"; +constexpr static llvm::StringLiteral kEndiannesKeyName = "dltest.endianness"; constexpr static llvm::StringLiteral kAllocaKeyName = "dltest.alloca_memory_space"; constexpr static llvm::StringLiteral kProgramKeyName = @@ -73,6 +74,9 @@ struct CustomDataLayoutSpec } DataLayoutEntryListRef getEntries() const { return getImpl()->entries; } LogicalResult verifySpec(Location loc) { return success(); } + StringAttr getEndiannessIdentifier(MLIRContext *context) const { + return Builder(context).getStringAttr(kEndiannesKeyName); + } StringAttr getAllocaMemorySpaceIdentifier(MLIRContext *context) const { return Builder(context).getStringAttr(kAllocaKeyName); } @@ -130,6 +134,15 @@ struct SingleQueryType return 4; } + Attribute getEndianness(DataLayoutEntryInterface entry) { + static bool executed = false; + if (executed) + llvm::report_fatal_error("repeated call"); + + executed = true; + return Attribute(); + } + Attribute getAllocaMemorySpace(DataLayoutEntryInterface entry) { static bool executed = false; if (executed) @@ -317,6 +330,7 @@ module {} EXPECT_EQ(layout.getTypePreferredAlignment(IntegerType::get(&ctx, 42)), 8u); EXPECT_EQ(layout.getTypePreferredAlignment(Float16Type::get(&ctx)), 2u); + EXPECT_EQ(layout.getEndianness(), Attribute()); EXPECT_EQ(layout.getAllocaMemorySpace(), Attribute()); EXPECT_EQ(layout.getProgramMemorySpace(), Attribute()); EXPECT_EQ(layout.getGlobalMemorySpace(), Attribute()); @@ -348,6 +362,7 @@ TEST(DataLayout, NullSpec) { EXPECT_EQ(layout.getTypeIndexBitwidth(Float16Type::get(&ctx)), std::nullopt); EXPECT_EQ(layout.getTypeIndexBitwidth(IndexType::get(&ctx)), 64u); + EXPECT_EQ(layout.getEndianness(), Attribute()); EXPECT_EQ(layout.getAllocaMemorySpace(), Attribute()); EXPECT_EQ(layout.getProgramMemorySpace(), Attribute()); EXPECT_EQ(layout.getGlobalMemorySpace(), Attribute()); @@ -378,6 +393,7 @@ TEST(DataLayout, EmptySpec) { EXPECT_EQ(layout.getTypeIndexBitwidth(Float16Type::get(&ctx)), std::nullopt); EXPECT_EQ(layout.getTypeIndexBitwidth(IndexType::get(&ctx)), 64u); + EXPECT_EQ(layout.getEndianness(), Attribute()); EXPECT_EQ(layout.getAllocaMemorySpace(), Attribute()); EXPECT_EQ(layout.getProgramMemorySpace(), Attribute()); EXPECT_EQ(layout.getGlobalMemorySpace(), Attribute()); @@ -390,6 +406,7 @@ TEST(DataLayout, SpecWithEntries) { #dlti.dl_entry<i42, 5>, #dlti.dl_entry<i16, 6>, #dlti.dl_entry<index, 42>, + #dlti.dl_entry<"dltest.endianness", "little">, #dlti.dl_entry<"dltest.alloca_memory_space", 5 : i32>, #dlti.dl_entry<"dltest.program_memory_space", 3 : i32>, #dlti.dl_entry<"dltest.global_memory_space", 2 : i32>, @@ -425,6 +442,7 @@ TEST(DataLayout, SpecWithEntries) { EXPECT_EQ(layout.getTypePreferredAlignment(IntegerType::get(&ctx, 32)), 64u); EXPECT_EQ(layout.getTypePreferredAlignment(Float32Type::get(&ctx)), 64u); + EXPECT_EQ(layout.getEndianness(), Builder(&ctx).getStringAttr("little")); EXPECT_EQ(layout.getAllocaMemorySpace(), Builder(&ctx).getI32IntegerAttr(5)); EXPECT_EQ(layout.getProgramMemorySpace(), Builder(&ctx).getI32IntegerAttr(3)); EXPECT_EQ(layout.getGlobalMemorySpace(), Builder(&ctx).getI32IntegerAttr(2)); |