From fab8e2732a0052fc4c0ee6b902c95c3b18f6c1d0 Mon Sep 17 00:00:00 2001 From: Justin Rosner Date: Tue, 14 Jul 2026 18:45:25 +0000 Subject: [PATCH 1/5] Fix cached db error --- src/targets/gpu/compile_ops.cpp | 64 ++++++++++++++-------- test/verify/test_conv_add_tune.cpp | 85 ++++++++++++++++++++++++++++++ 2 files changed, 127 insertions(+), 22 deletions(-) diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index 431666fe54c..a5f334fb9cf 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -44,6 +44,7 @@ #include #include #include +#include #include #include #include @@ -314,6 +315,16 @@ struct compile_plan { config = get_tuning_config(*ctx, ins, preop, exhaustive); } + + bool is_cached_solution_applicable(const value& solution) const + { + if(preop.name() != "gpu::mlir_op" or solution.if_string() == nullptr) + return true; + + const auto& modules = ins->module_inputs(); + return modules.empty() or is_module_fusible(*modules.front(), *ctx, solution); + } + template void insert_compiles(Vector& compiles, const value& solution, std::size_t i) { @@ -345,35 +356,44 @@ struct compile_plan if(auto sol = ctx->get_problem_cache().get(preop.name(), problem)) { const auto& solution = sol.value(); - // No solution yet until benchmarked so skip for now + // A null cache entry means this problem has been marked for benchmarking, + // but no winning solution has been recorded yet. if(solution.is_null()) return; + if(is_cached_solution_applicable(solution)) + { + results.resize(1); + insert_compiles(compiles, solution, 0); + return; + } + if(value_of(MIGRAPHX_TRACE_BENCHMARKING{}) > 0) + std::cout << "Ignoring cached solution for " << preop.name() + << " because it is not applicable to the fused module" + << std::endl; + } + + // No usable cached solution: choose a configured solution directly or benchmark + // the candidates and cache the winner later. + const auto& solutions = config->solutions; + if(solutions.empty()) + MIGRAPHX_THROW("No solutions provided for " + preop.name() + " with " + + problem_string() + "\n\n" + print_modules()); + const bool dump_mxr = + not string_value_of(MIGRAPHX_GPU_DUMP_BENCHMARK_MXR{}).empty(); + if(enabled(MIGRAPHX_SKIP_BENCHMARKING{}) or + (ctx->is_cross_compile() and not dump_mxr) or solutions.size() == 1) + { + ctx->get_problem_cache().insert(preop.name(), problem, solutions.front()); results.resize(1); - insert_compiles(compiles, solution, 0); + insert_compiles(compiles, solutions.front(), 0); } else { - const auto& solutions = config->solutions; - if(solutions.empty()) - MIGRAPHX_THROW("No solutions provided for " + preop.name() + " with " + - problem_string() + "\n\n" + print_modules()); - const bool dump_mxr = - not string_value_of(MIGRAPHX_GPU_DUMP_BENCHMARK_MXR{}).empty(); - if(enabled(MIGRAPHX_SKIP_BENCHMARKING{}) or - (ctx->is_cross_compile() and not dump_mxr) or solutions.size() == 1) - { - ctx->get_problem_cache().insert(preop.name(), problem, solutions.front()); - results.resize(1); - insert_compiles(compiles, solutions.front(), 0); - } - else + ctx->get_problem_cache().mark(preop.name(), problem); + results.resize(solutions.size()); + for(auto i : range(solutions.size())) { - ctx->get_problem_cache().mark(preop.name(), problem); - results.resize(solutions.size()); - for(auto i : range(solutions.size())) - { - insert_compiles(compiles, solutions[i], i); - } + insert_compiles(compiles, solutions[i], i); } } } diff --git a/test/verify/test_conv_add_tune.cpp b/test/verify/test_conv_add_tune.cpp index e208b2a872c..a09f9c8ba53 100644 --- a/test/verify/test_conv_add_tune.cpp +++ b/test/verify/test_conv_add_tune.cpp @@ -24,9 +24,15 @@ #include "verify_program.hpp" #include +#include #include +#include +#include #include #include +#include +#include +#include template struct test_conv_add_tune : verify_program> @@ -71,6 +77,85 @@ struct test_conv_add_tune : verify_program> std::string section() const { return "conv"; } }; +struct test_conv_add_tune_bad_perf_config_cache + : verify_program +{ + test_conv_add_tune_bad_perf_config_cache() + { + static migraphx::tmp_dir td{"conv_add_tune_problem_cache"}; + auto cache_path = td.path / "problem_cache.json"; + const std::string problem_config = + "gfx1100\t48\t1\tconv -F 1 -f GNC01 -I NGC01 -O NGC01 -n 1 -c 256 -H 32 -W " + "32 -k 256 -y 3 -x 3 -p 1 -q 1 -u 1 -v 1 -l 1 -j 1 -g 1"; + const std::string perf_config = + "gemm:v3:64,128,16,1,1,4,0,4,1,0,0,-1,-1,-1,-1,-1"; + migraphx::value cache_entry = migraphx::value::array{}; + cache_entry.push_back( + {{"name", std::string{"gpu::mlir_op"}}, {"problem", problem_config}}); + cache_entry.push_back(perf_config); + + migraphx::value cache = migraphx::value::array{}; + cache.push_back(cache_entry); + migraphx::write_string(cache_path, migraphx::to_pretty_json_string(cache)); + setenv("MIGRAPHX_PROBLEM_CACHE", cache_path.string().c_str(), 1); + } + + migraphx::program create_program() const + { + migraphx::program p; + auto* mm = p.get_main_module(); + + migraphx::shape input_shape{migraphx::shape::float_type, {1, 256, 32, 32}}; + auto x = mm->add_parameter("x", input_shape); + auto w = mm->add_parameter("w", {migraphx::shape::float_type, {256, 256, 3, 3}}); + auto bias = mm->add_parameter("bias", {migraphx::shape::float_type, {256}}); + auto zero = mm->add_literal( + migraphx::literal{{migraphx::shape::float_type, {1}}, {0.0f}}); + auto scale = mm->add_literal( + migraphx::literal{{migraphx::shape::float_type, {1}}, {0.2f}}); + + auto conv = + mm->add_instruction(migraphx::make_op("convolution", + {{"padding", {1, 1, 1, 1}}, + {"stride", {1, 1}}, + {"dilation", {1, 1}}}), + x, + w); + std::vector lens = {1, 256, 32, 32}; + auto bias_bcast = mm->add_instruction( + migraphx::make_op("broadcast", {{"axis", 1}, {"out_lens", lens}}), bias); + auto add = mm->add_instruction(migraphx::make_op("add"), conv, bias_bcast); + auto zero_bcast = + mm->add_instruction(migraphx::make_op("multibroadcast", {{"out_lens", lens}}), zero); + auto greater = mm->add_instruction(migraphx::make_op("greater"), add, zero_bcast); + auto scale_bcast = + mm->add_instruction(migraphx::make_op("multibroadcast", {{"out_lens", lens}}), scale); + auto mul = mm->add_instruction(migraphx::make_op("mul"), add, scale_bcast); + auto cond = + mm->add_instruction(migraphx::make_op("convert", + {{"target_type", migraphx::shape::bool_type}}), + greater); + auto where = mm->add_instruction(migraphx::make_op("where"), cond, add, mul); + auto reshape1 = + mm->add_instruction(migraphx::make_op("reshape", {{"dims", {1, 256, 16, 2, 16, 2}}}), + where); + auto transpose = + mm->add_instruction(migraphx::make_op("transpose", + {{"permutation", {5, 3, 0, 1, 2, 4}}}), + reshape1); + mm->add_instruction(migraphx::make_op("reshape", {{"dims", {4, 256, 16, 16}}}), + transpose); + return p; + } + + migraphx::compile_options get_compile_options() const + { + return migraphx::compile_options{}; + } + + std::string section() const { return "conv"; } +}; + template struct test_conv_add_tune; template struct test_conv_add_tune; template struct test_conv_add_tune; From 2c3864e573b46bc93ca3ca0f402996560d088e41 Mon Sep 17 00:00:00 2001 From: Justin Rosner Date: Tue, 14 Jul 2026 18:56:58 +0000 Subject: [PATCH 2/5] Changelog update --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 66525c60007..caaf2a1865f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,6 +83,7 @@ Full documentation for MIGraphX is available at * Fixed `QLinearConv` parsing for models with a bias and per-tensor weight quantization, which previously threw `same_dims: dequantizelinear: Dimensions do not match` (e.g. `resnet50_int8`); the bias scale is now broadcast to the bias shape before dequantizing. * Fixed the GPU problem cache failing to find entries after reload for pooling operator, resulting in redundant re-benchmarking when using a saved `MIGRAPHX_PROBLEM_CACHE`. * Fixed `slice_concat_gather` matcher and interaction between same table and cross table gather fusions(#5038). +* Fixed `gpu::mlir_op` compilation failures caused by reusing a cached `perfConfig` that is not applicable to a fused MLIR module. ### Optimized * Optimized flash decoding recombination in `fuse_attention` to use the exp-normalize form (#5090). From 525cf6f91f3351a4f8b18afa580f2ef8bb1a66d8 Mon Sep 17 00:00:00 2001 From: Justin Rosner Date: Wed, 15 Jul 2026 00:58:14 +0000 Subject: [PATCH 3/5] New approach --- CHANGELOG.md | 2 +- src/targets/gpu/compile_ops.cpp | 64 ++++++---------- src/targets/gpu/jit/mlir.cpp | 107 +++++++++++++++++++++++--- test/verify/test_conv_add_tune.cpp | 118 ++++++++++------------------- 4 files changed, 158 insertions(+), 133 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index caaf2a1865f..edbc8b93963 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -83,7 +83,7 @@ Full documentation for MIGraphX is available at * Fixed `QLinearConv` parsing for models with a bias and per-tensor weight quantization, which previously threw `same_dims: dequantizelinear: Dimensions do not match` (e.g. `resnet50_int8`); the bias scale is now broadcast to the bias shape before dequantizing. * Fixed the GPU problem cache failing to find entries after reload for pooling operator, resulting in redundant re-benchmarking when using a saved `MIGRAPHX_PROBLEM_CACHE`. * Fixed `slice_concat_gather` matcher and interaction between same table and cross table gather fusions(#5038). -* Fixed `gpu::mlir_op` compilation failures caused by reusing a cached `perfConfig` that is not applicable to a fused MLIR module. +* Fixed `gpu::mlir_op` compilation failures for convolution and pointwise fusions followed by layout operations by splitting them into MLIR, pointwise, and layout-copy kernels when needed. ### Optimized * Optimized flash decoding recombination in `fuse_attention` to use the exp-normalize form (#5090). diff --git a/src/targets/gpu/compile_ops.cpp b/src/targets/gpu/compile_ops.cpp index a5f334fb9cf..431666fe54c 100644 --- a/src/targets/gpu/compile_ops.cpp +++ b/src/targets/gpu/compile_ops.cpp @@ -44,7 +44,6 @@ #include #include #include -#include #include #include #include @@ -315,16 +314,6 @@ struct compile_plan { config = get_tuning_config(*ctx, ins, preop, exhaustive); } - - bool is_cached_solution_applicable(const value& solution) const - { - if(preop.name() != "gpu::mlir_op" or solution.if_string() == nullptr) - return true; - - const auto& modules = ins->module_inputs(); - return modules.empty() or is_module_fusible(*modules.front(), *ctx, solution); - } - template void insert_compiles(Vector& compiles, const value& solution, std::size_t i) { @@ -356,44 +345,35 @@ struct compile_plan if(auto sol = ctx->get_problem_cache().get(preop.name(), problem)) { const auto& solution = sol.value(); - // A null cache entry means this problem has been marked for benchmarking, - // but no winning solution has been recorded yet. + // No solution yet until benchmarked so skip for now if(solution.is_null()) return; - if(is_cached_solution_applicable(solution)) - { - results.resize(1); - insert_compiles(compiles, solution, 0); - return; - } - if(value_of(MIGRAPHX_TRACE_BENCHMARKING{}) > 0) - std::cout << "Ignoring cached solution for " << preop.name() - << " because it is not applicable to the fused module" - << std::endl; - } - - // No usable cached solution: choose a configured solution directly or benchmark - // the candidates and cache the winner later. - const auto& solutions = config->solutions; - if(solutions.empty()) - MIGRAPHX_THROW("No solutions provided for " + preop.name() + " with " + - problem_string() + "\n\n" + print_modules()); - const bool dump_mxr = - not string_value_of(MIGRAPHX_GPU_DUMP_BENCHMARK_MXR{}).empty(); - if(enabled(MIGRAPHX_SKIP_BENCHMARKING{}) or - (ctx->is_cross_compile() and not dump_mxr) or solutions.size() == 1) - { - ctx->get_problem_cache().insert(preop.name(), problem, solutions.front()); results.resize(1); - insert_compiles(compiles, solutions.front(), 0); + insert_compiles(compiles, solution, 0); } else { - ctx->get_problem_cache().mark(preop.name(), problem); - results.resize(solutions.size()); - for(auto i : range(solutions.size())) + const auto& solutions = config->solutions; + if(solutions.empty()) + MIGRAPHX_THROW("No solutions provided for " + preop.name() + " with " + + problem_string() + "\n\n" + print_modules()); + const bool dump_mxr = + not string_value_of(MIGRAPHX_GPU_DUMP_BENCHMARK_MXR{}).empty(); + if(enabled(MIGRAPHX_SKIP_BENCHMARKING{}) or + (ctx->is_cross_compile() and not dump_mxr) or solutions.size() == 1) + { + ctx->get_problem_cache().insert(preop.name(), problem, solutions.front()); + results.resize(1); + insert_compiles(compiles, solutions.front(), 0); + } + else { - insert_compiles(compiles, solutions[i], i); + ctx->get_problem_cache().mark(preop.name(), problem); + results.resize(solutions.size()); + for(auto i : range(solutions.size())) + { + insert_compiles(compiles, solutions[i], i); + } } } } diff --git a/src/targets/gpu/jit/mlir.cpp b/src/targets/gpu/jit/mlir.cpp index 7bc73a80220..556e7f818ad 100644 --- a/src/targets/gpu/jit/mlir.cpp +++ b/src/targets/gpu/jit/mlir.cpp @@ -135,6 +135,26 @@ static instruction_ref find_final_split(instruction_ref split_ins) return *it; } +static optional find_layout_tail_split(instruction_ref pointwise_ins) +{ + auto output_path_range = get_output_path(pointwise_ins); + std::vector output_path(output_path_range.begin(), output_path_range.end()); + if(output_path.size() < 2) + return nullopt; + auto is_layout = [](instruction_ref ins) { + return contains({"flatten", "reshape", "reshape_lazy", "squeeze", "transpose", "unsqueeze"}, + ins->name()); + }; + auto it = std::find_if(std::next(output_path.begin()), output_path.end(), is_layout); + if(it == output_path.end()) + return nullopt; + if(not std::all_of(it, output_path.end(), [&](instruction_ref ins) { + return is_layout(ins) or ins->name() == "@return"; + })) + return nullopt; + return *std::prev(it); +} + struct mlir_compiler : compiler { std::vector names() const { return {"gpu::mlir_op"}; } @@ -199,6 +219,29 @@ struct mlir_compiler : compiler } } + mlir_code_object compile_mlir_part(context& ctx, + const module_with_inputs& mwi, + const value& solution) const + { + auto input_shapes = to_shapes(mwi.inputs); + input_shapes.push_back(mwi.mod.get_output_shapes().front()); + return compile_mlir(ctx, mwi.mod, input_shapes, solution); + } + + code_object_op compile_pointwise_part(context& ctx, module_with_inputs& mwi) const + { + auto input_shapes = to_shapes(mwi.inputs); + if(mwi.mod.get_output_shapes().size() == 1) + { + input_shapes.push_back(mwi.mod.get_output_shapes().front()); + } + else + { + input_shapes.push_back(shape{mwi.mod.get_output_shapes()}); + } + return compile_pointwise_module(ctx, input_shapes, &mwi.mod); + } + compiler_replace compile(context& ctx, instruction_ref ins, const operation&, const value& solution) const { @@ -219,15 +262,36 @@ struct mlir_compiler : compiler auto input_args = ins->inputs(); // remove alloc buffer input_args.pop_back(); - auto split_ins = find_final_split(gemm_like_ins); + auto tail_split = find_layout_tail_split(pointwise_ins); + auto split_ins = tail_split.has_value() ? gemm_like_ins : find_final_split(gemm_like_ins); std::array mod_splits = smod->split(input_args, {split_ins}); - auto dot_mlir_inputs = to_shapes(mod_splits[0].inputs); - // add alloc for the gemm output - dot_mlir_inputs.push_back(mod_splits[0].mod.get_output_shapes().front()); - mlir_code_object cop1 = compile_mlir(ctx, mod_splits[0].mod, dot_mlir_inputs, solution); - auto pw_shapes = to_shapes(mod_splits[1].inputs); - pw_shapes.push_back(ins->get_shape()); - auto cop2 = compile_pointwise_module(ctx, pw_shapes, &mod_splits[1].mod); + if(not is_module_fusible(mod_splits[0].mod, ctx, solution)) + { + split_ins = gemm_like_ins; + mod_splits = smod->split(input_args, {split_ins}); + } + if(tail_split.has_value()) + { + auto mod_splits3 = smod->split(input_args, {split_ins}, {tail_split.value()}); + auto copy_input_shape = mod_splits3[2].mod.get_output_shapes().front(); + auto copy_cop = any_cast( + gpu::compile_op("hip::copy", + ctx, + {copy_input_shape, ins->inputs().back()->get_shape()}, + {{"lambda", "[](auto x) { return make_tuple(x); }"}, + {"kernel", "hip_copy_kernel"}})); + std::vector cops = { + compile_mlir_part(ctx, mod_splits3[0], solution), + mlir_code_object{compile_pointwise_part(ctx, mod_splits3[1])}, + mlir_code_object{copy_cop}}; + std::array mods = {std::move(mod_splits3[0]), + std::move(mod_splits3[1])}; + return insert(cops, mods, ins, split_ins); + } + + auto cop1 = compile_mlir_part(ctx, mod_splits[0], solution); + auto cop2 = compile_pointwise_part(ctx, mod_splits[1]); + assert(cop2.expected_inputs.back() == ins->get_shape()); std::vector cops = {cop1, mlir_code_object{cop2}}; return insert(cops, mod_splits, ins, split_ins); } @@ -345,8 +409,21 @@ struct mlir_compiler : compiler insert_mlir(m, ins, any_cast(ops[0]), dot_inputs_updated); auto pwm = mods[1]; pwm.replace(split_ins, mlir_ins); + const bool has_copy_tail = ops.size() == 3; auto pw_inputs = pwm.inputs; - pw_inputs.push_back(ins->inputs().back()); + if(has_copy_tail) + { + auto pw_alloc = m.insert_instruction( + ins, + migraphx::make_op( + "hip::allocate", + {{"shape", to_value(mods[1].mod.get_output_shapes().front())}})); + pw_inputs.push_back(pw_alloc); + } + else + { + pw_inputs.push_back(ins->inputs().back()); + } std::vector pw_inputs_updated; std::transform(pw_inputs.begin(), pw_inputs.end(), @@ -361,7 +438,17 @@ struct mlir_compiler : compiler }); auto pw_ins = insert_mlir(m, ins, any_cast(ops[1]), pw_inputs_updated); - return m.replace_instruction(ins, pw_ins); + if(not has_copy_tail) + return m.replace_instruction(ins, pw_ins); + + auto copy_input_shape = any_cast(ops[2]).expected_inputs.front(); + auto copy_input = m.insert_instruction( + ins, + migraphx::make_op("as_shape", {{"shape", to_value(copy_input_shape)}}), + pw_ins); + auto copy_ins = m.insert_instruction( + ins, any_cast(ops[2]), copy_input, ins->inputs().back()); + return m.replace_instruction(ins, copy_ins); }}; } diff --git a/test/verify/test_conv_add_tune.cpp b/test/verify/test_conv_add_tune.cpp index a09f9c8ba53..87387db1baf 100644 --- a/test/verify/test_conv_add_tune.cpp +++ b/test/verify/test_conv_add_tune.cpp @@ -24,15 +24,9 @@ #include "verify_program.hpp" #include -#include #include -#include -#include #include #include -#include -#include -#include template struct test_conv_add_tune : verify_program> @@ -77,89 +71,53 @@ struct test_conv_add_tune : verify_program> std::string section() const { return "conv"; } }; -struct test_conv_add_tune_bad_perf_config_cache - : verify_program -{ - test_conv_add_tune_bad_perf_config_cache() - { - static migraphx::tmp_dir td{"conv_add_tune_problem_cache"}; - auto cache_path = td.path / "problem_cache.json"; - const std::string problem_config = - "gfx1100\t48\t1\tconv -F 1 -f GNC01 -I NGC01 -O NGC01 -n 1 -c 256 -H 32 -W " - "32 -k 256 -y 3 -x 3 -p 1 -q 1 -u 1 -v 1 -l 1 -j 1 -g 1"; - const std::string perf_config = - "gemm:v3:64,128,16,1,1,4,0,4,1,0,0,-1,-1,-1,-1,-1"; - migraphx::value cache_entry = migraphx::value::array{}; - cache_entry.push_back( - {{"name", std::string{"gpu::mlir_op"}}, {"problem", problem_config}}); - cache_entry.push_back(perf_config); - - migraphx::value cache = migraphx::value::array{}; - cache.push_back(cache_entry); - migraphx::write_string(cache_path, migraphx::to_pretty_json_string(cache)); - setenv("MIGRAPHX_PROBLEM_CACHE", cache_path.string().c_str(), 1); - } +template struct test_conv_add_tune; +template struct test_conv_add_tune; +template struct test_conv_add_tune; +template struct test_conv_add_tune; +template struct test_conv_add_tune; +template struct test_conv_add_tune; +template struct test_conv_add_tune; +struct test_conv_add_layout_tune : verify_program +{ + // This covers split-k perfConfigs that reject conv+pointwise+layout fusion and require + // compiling the fused MLIR op as conv, pointwise, and layout-copy kernels. migraphx::program create_program() const { migraphx::program p; auto* mm = p.get_main_module(); + auto x1 = mm->add_parameter("x1", {migraphx::shape::half_type, {1, 256, 16, 16}}); + auto w1 = + mm->add_literal(migraphx::generate_literal({migraphx::shape::half_type, {1, 256, 3, 2}}, 1)); + auto x2 = mm->add_parameter("x2", {migraphx::shape::half_type, {1, 256, 16, 16}}); + auto w2 = + mm->add_literal(migraphx::generate_literal({migraphx::shape::half_type, {1, 256, 3, 2}}, 1)); - migraphx::shape input_shape{migraphx::shape::float_type, {1, 256, 32, 32}}; - auto x = mm->add_parameter("x", input_shape); - auto w = mm->add_parameter("w", {migraphx::shape::float_type, {256, 256, 3, 3}}); - auto bias = mm->add_parameter("bias", {migraphx::shape::float_type, {256}}); - auto zero = mm->add_literal( - migraphx::literal{{migraphx::shape::float_type, {1}}, {0.0f}}); - auto scale = mm->add_literal( - migraphx::literal{{migraphx::shape::float_type, {1}}, {0.2f}}); - - auto conv = - mm->add_instruction(migraphx::make_op("convolution", - {{"padding", {1, 1, 1, 1}}, + auto conv1 = mm->add_instruction( + migraphx::make_op("convolution", {{"padding", {1, 1, 1, 0}}, {"stride", {2, 2}}}), + x1, + w1); + auto pooling = + mm->add_instruction(migraphx::make_op("pooling", + {{"mode", migraphx::op::pooling_mode::average}, + {"padding", {1, 1, 1, 1}}, {"stride", {1, 1}}, - {"dilation", {1, 1}}}), - x, - w); - std::vector lens = {1, 256, 32, 32}; - auto bias_bcast = mm->add_instruction( - migraphx::make_op("broadcast", {{"axis", 1}, {"out_lens", lens}}), bias); - auto add = mm->add_instruction(migraphx::make_op("add"), conv, bias_bcast); - auto zero_bcast = - mm->add_instruction(migraphx::make_op("multibroadcast", {{"out_lens", lens}}), zero); - auto greater = mm->add_instruction(migraphx::make_op("greater"), add, zero_bcast); - auto scale_bcast = - mm->add_instruction(migraphx::make_op("multibroadcast", {{"out_lens", lens}}), scale); - auto mul = mm->add_instruction(migraphx::make_op("mul"), add, scale_bcast); - auto cond = - mm->add_instruction(migraphx::make_op("convert", - {{"target_type", migraphx::shape::bool_type}}), - greater); - auto where = mm->add_instruction(migraphx::make_op("where"), cond, add, mul); - auto reshape1 = - mm->add_instruction(migraphx::make_op("reshape", {{"dims", {1, 256, 16, 2, 16, 2}}}), - where); - auto transpose = - mm->add_instruction(migraphx::make_op("transpose", - {{"permutation", {5, 3, 0, 1, 2, 4}}}), - reshape1); - mm->add_instruction(migraphx::make_op("reshape", {{"dims", {4, 256, 16, 16}}}), - transpose); - return p; - } + {"lengths", {3, 3}}, + {"count_include_pad", false}}), + conv1); + auto conv2 = mm->add_instruction( + migraphx::make_op("convolution", {{"padding", {1, 1, 1, 0}}, {"stride", {2, 2}}}), + x2, + w2); + auto add = mm->add_instruction(migraphx::make_op("add"), pooling, conv2); - migraphx::compile_options get_compile_options() const - { - return migraphx::compile_options{}; + auto reshape = mm->add_instruction( + migraphx::make_op("reshape_lazy", {{"dims", {1, 1, 4, 2, 8}}}), add); + mm->add_instruction(migraphx::make_op("transpose", {{"permutation", {0, 1, 3, 2, 4}}}), + reshape); + return p; } std::string section() const { return "conv"; } }; - -template struct test_conv_add_tune; -template struct test_conv_add_tune; -template struct test_conv_add_tune; -template struct test_conv_add_tune; -template struct test_conv_add_tune; -template struct test_conv_add_tune; -template struct test_conv_add_tune; From b28759c951f336a0504ae4c170ac9c484647e17c Mon Sep 17 00:00:00 2001 From: Justin Rosner Date: Wed, 15 Jul 2026 00:58:51 +0000 Subject: [PATCH 4/5] Formatting --- src/targets/gpu/jit/mlir.cpp | 24 ++++++++++++------------ test/verify/test_conv_add_tune.cpp | 8 ++++---- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/targets/gpu/jit/mlir.cpp b/src/targets/gpu/jit/mlir.cpp index 556e7f818ad..e74c2a8537c 100644 --- a/src/targets/gpu/jit/mlir.cpp +++ b/src/targets/gpu/jit/mlir.cpp @@ -219,9 +219,8 @@ struct mlir_compiler : compiler } } - mlir_code_object compile_mlir_part(context& ctx, - const module_with_inputs& mwi, - const value& solution) const + mlir_code_object + compile_mlir_part(context& ctx, const module_with_inputs& mwi, const value& solution) const { auto input_shapes = to_shapes(mwi.inputs); input_shapes.push_back(mwi.mod.get_output_shapes().front()); @@ -262,8 +261,9 @@ struct mlir_compiler : compiler auto input_args = ins->inputs(); // remove alloc buffer input_args.pop_back(); - auto tail_split = find_layout_tail_split(pointwise_ins); - auto split_ins = tail_split.has_value() ? gemm_like_ins : find_final_split(gemm_like_ins); + auto tail_split = find_layout_tail_split(pointwise_ins); + auto split_ins = + tail_split.has_value() ? gemm_like_ins : find_final_split(gemm_like_ins); std::array mod_splits = smod->split(input_args, {split_ins}); if(not is_module_fusible(mod_splits[0].mod, ctx, solution)) { @@ -272,14 +272,14 @@ struct mlir_compiler : compiler } if(tail_split.has_value()) { - auto mod_splits3 = smod->split(input_args, {split_ins}, {tail_split.value()}); - auto copy_input_shape = mod_splits3[2].mod.get_output_shapes().front(); - auto copy_cop = any_cast( + auto mod_splits3 = smod->split(input_args, {split_ins}, {tail_split.value()}); + auto copy_input_shape = mod_splits3[2].mod.get_output_shapes().front(); + auto copy_cop = any_cast( gpu::compile_op("hip::copy", ctx, - {copy_input_shape, ins->inputs().back()->get_shape()}, - {{"lambda", "[](auto x) { return make_tuple(x); }"}, - {"kernel", "hip_copy_kernel"}})); + {copy_input_shape, ins->inputs().back()->get_shape()}, + {{"lambda", "[](auto x) { return make_tuple(x); }"}, + {"kernel", "hip_copy_kernel"}})); std::vector cops = { compile_mlir_part(ctx, mod_splits3[0], solution), mlir_code_object{compile_pointwise_part(ctx, mod_splits3[1])}, @@ -442,7 +442,7 @@ struct mlir_compiler : compiler return m.replace_instruction(ins, pw_ins); auto copy_input_shape = any_cast(ops[2]).expected_inputs.front(); - auto copy_input = m.insert_instruction( + auto copy_input = m.insert_instruction( ins, migraphx::make_op("as_shape", {{"shape", to_value(copy_input_shape)}}), pw_ins); diff --git a/test/verify/test_conv_add_tune.cpp b/test/verify/test_conv_add_tune.cpp index 87387db1baf..5108bfebf1a 100644 --- a/test/verify/test_conv_add_tune.cpp +++ b/test/verify/test_conv_add_tune.cpp @@ -88,11 +88,11 @@ struct test_conv_add_layout_tune : verify_program migraphx::program p; auto* mm = p.get_main_module(); auto x1 = mm->add_parameter("x1", {migraphx::shape::half_type, {1, 256, 16, 16}}); - auto w1 = - mm->add_literal(migraphx::generate_literal({migraphx::shape::half_type, {1, 256, 3, 2}}, 1)); + auto w1 = mm->add_literal( + migraphx::generate_literal({migraphx::shape::half_type, {1, 256, 3, 2}}, 1)); auto x2 = mm->add_parameter("x2", {migraphx::shape::half_type, {1, 256, 16, 16}}); - auto w2 = - mm->add_literal(migraphx::generate_literal({migraphx::shape::half_type, {1, 256, 3, 2}}, 1)); + auto w2 = mm->add_literal( + migraphx::generate_literal({migraphx::shape::half_type, {1, 256, 3, 2}}, 1)); auto conv1 = mm->add_instruction( migraphx::make_op("convolution", {{"padding", {1, 1, 1, 0}}, {"stride", {2, 2}}}), From d118e6fc4b867118592f0f8eaf44adc186e0885c Mon Sep 17 00:00:00 2001 From: Justin Rosner Date: Mon, 20 Jul 2026 20:22:02 +0000 Subject: [PATCH 5/5] Update unit tests --- .../test_conv_add_reshape_lazy_transpose.cpp | 64 +++++++++++++++++++ test/verify/test_conv_add_tune.cpp | 42 ------------ .../test_conv_relu_reshape_lazy_transpose.cpp | 62 ++++++++++++++++++ 3 files changed, 126 insertions(+), 42 deletions(-) create mode 100644 test/verify/test_conv_add_reshape_lazy_transpose.cpp create mode 100644 test/verify/test_conv_relu_reshape_lazy_transpose.cpp diff --git a/test/verify/test_conv_add_reshape_lazy_transpose.cpp b/test/verify/test_conv_add_reshape_lazy_transpose.cpp new file mode 100644 index 00000000000..3d0b10b0be8 --- /dev/null +++ b/test/verify/test_conv_add_reshape_lazy_transpose.cpp @@ -0,0 +1,64 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "verify_program.hpp" +#include +#include +#include + +struct test_conv_add_reshape_lazy_transpose + : verify_program +{ + // This covers split-k perfConfigs that reject conv+pointwise+layout fusion and require + // compiling the fused MLIR op as conv, pointwise, and layout-copy kernels. + migraphx::program create_program() const + { + migraphx::program p; + auto* mm = p.get_main_module(); + auto input = mm->add_parameter("x", {migraphx::shape::half_type, {1, 256, 16, 16}}); + auto weight = mm->add_literal( + migraphx::generate_literal({migraphx::shape::half_type, {1, 256, 3, 2}}, 1)); + auto y = mm->add_parameter("y", {migraphx::shape::half_type, {1, 1, 8, 8}}); + + auto conv = mm->add_instruction( + migraphx::make_op("convolution", {{"padding", {1, 1, 1, 0}}, {"stride", {2, 2}}}), + input, + weight); + auto add = mm->add_instruction(migraphx::make_op("add"), conv, y); + + auto reshape = mm->add_instruction( + migraphx::make_op("reshape_lazy", {{"dims", {1, 1, 4, 2, 8}}}), add); + mm->add_instruction(migraphx::make_op("transpose", {{"permutation", {0, 1, 3, 2, 4}}}), + reshape); + return p; + } + + // Turn on Exhaustive-tune to enable split-k perf-configs from MLIR + migraphx::compile_options get_compile_options() const + { + return migraphx::compile_options{.exhaustive_tune = true}; + } + + std::string section() const { return "conv"; } +}; diff --git a/test/verify/test_conv_add_tune.cpp b/test/verify/test_conv_add_tune.cpp index 5108bfebf1a..b17b4265718 100644 --- a/test/verify/test_conv_add_tune.cpp +++ b/test/verify/test_conv_add_tune.cpp @@ -79,45 +79,3 @@ template struct test_conv_add_tune; template struct test_conv_add_tune; template struct test_conv_add_tune; -struct test_conv_add_layout_tune : verify_program -{ - // This covers split-k perfConfigs that reject conv+pointwise+layout fusion and require - // compiling the fused MLIR op as conv, pointwise, and layout-copy kernels. - migraphx::program create_program() const - { - migraphx::program p; - auto* mm = p.get_main_module(); - auto x1 = mm->add_parameter("x1", {migraphx::shape::half_type, {1, 256, 16, 16}}); - auto w1 = mm->add_literal( - migraphx::generate_literal({migraphx::shape::half_type, {1, 256, 3, 2}}, 1)); - auto x2 = mm->add_parameter("x2", {migraphx::shape::half_type, {1, 256, 16, 16}}); - auto w2 = mm->add_literal( - migraphx::generate_literal({migraphx::shape::half_type, {1, 256, 3, 2}}, 1)); - - auto conv1 = mm->add_instruction( - migraphx::make_op("convolution", {{"padding", {1, 1, 1, 0}}, {"stride", {2, 2}}}), - x1, - w1); - auto pooling = - mm->add_instruction(migraphx::make_op("pooling", - {{"mode", migraphx::op::pooling_mode::average}, - {"padding", {1, 1, 1, 1}}, - {"stride", {1, 1}}, - {"lengths", {3, 3}}, - {"count_include_pad", false}}), - conv1); - auto conv2 = mm->add_instruction( - migraphx::make_op("convolution", {{"padding", {1, 1, 1, 0}}, {"stride", {2, 2}}}), - x2, - w2); - auto add = mm->add_instruction(migraphx::make_op("add"), pooling, conv2); - - auto reshape = mm->add_instruction( - migraphx::make_op("reshape_lazy", {{"dims", {1, 1, 4, 2, 8}}}), add); - mm->add_instruction(migraphx::make_op("transpose", {{"permutation", {0, 1, 3, 2, 4}}}), - reshape); - return p; - } - - std::string section() const { return "conv"; } -}; diff --git a/test/verify/test_conv_relu_reshape_lazy_transpose.cpp b/test/verify/test_conv_relu_reshape_lazy_transpose.cpp new file mode 100644 index 00000000000..9ac74465224 --- /dev/null +++ b/test/verify/test_conv_relu_reshape_lazy_transpose.cpp @@ -0,0 +1,62 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015-2026 Advanced Micro Devices, Inc. All rights reserved. + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + */ + +#include "verify_program.hpp" +#include +#include +#include + +struct test_conv_relu_reshape_lazy_transpose + : verify_program +{ + // This covers split-k perfConfigs that reject conv+unary-pointwise+layout fusion. + migraphx::program create_program() const + { + migraphx::program p; + auto* mm = p.get_main_module(); + auto input = mm->add_parameter("x", {migraphx::shape::half_type, {1, 256, 16, 16}}); + auto weight = mm->add_literal( + migraphx::generate_literal({migraphx::shape::half_type, {1, 256, 3, 2}}, 1)); + + auto conv = mm->add_instruction( + migraphx::make_op("convolution", {{"padding", {1, 1, 1, 0}}, {"stride", {2, 2}}}), + input, + weight); + auto relu = mm->add_instruction(migraphx::make_op("relu"), conv); + + auto reshape = mm->add_instruction( + migraphx::make_op("reshape_lazy", {{"dims", {1, 1, 4, 2, 8}}}), relu); + mm->add_instruction(migraphx::make_op("transpose", {{"permutation", {0, 1, 3, 2, 4}}}), + reshape); + return p; + } + + // Turn on Exhaustive-tune to enable split-k perf-configs from MLIR + migraphx::compile_options get_compile_options() const + { + return migraphx::compile_options{.exhaustive_tune = true}; + } + + std::string section() const { return "conv"; } +};