set(kernel_declare_file
    ${PADDLE_BINARY_DIR}/paddle/phi/kernels/declarations.h.tmp
    CACHE INTERNAL "declarations.h file")
set(kernel_declare_file_final
    ${PADDLE_BINARY_DIR}/paddle/phi/kernels/declarations.h)
file(
  WRITE ${kernel_declare_file}
  "// Generated by the paddle/phi/kernels/CMakeLists.txt.  DO NOT EDIT!\n\n#pragma once\n\n"
)
file(APPEND ${kernel_declare_file}
     "#include \"paddle/phi/core/kernel_registry.h\"\n\n")
set(kernel_declare_file_prune
    ${PADDLE_BINARY_DIR}/paddle/phi/kernels/declarations.h.prune
    CACHE INTERNAL "declarations.h file")

# phi functors and functions called by kernels
add_subdirectory(funcs)

# kernel autotune
add_subdirectory(autotune)

copy_if_different(${kernel_declare_file} ${kernel_declare_file_final})

file(GLOB kernel_h "*.h" "selected_rows/*.h" "sparse/*.h" "strings/*.h")
file(GLOB kernel_impl_h "impl/*.h" "selected_rows/impl/*.h")
file(GLOB kernel_primitive_h "primitive/*.h")

# fusion ops would be included here
file(
  GLOB kernel_cu
  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
  "gpudnn/*.cu"
  "kps/*.cu"
  "stride/*.cu"
  "legacy/kps/*.cu"
  "legacy/gpu/*.cu"
  "selected_rows/gpu/*.cu"
  "sparse/gpu/*.cu"
  "strings/gpu/*.cu"
  "fusion/fp8_gemm/fp8_gemm_with_cublasLt/*.cu"
  "fusion/gpu/*.cu")

file(
  GLOB kernel_gpu
  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
  "gpu/*.cu" "gpu/*.cu.cc")

if(APPLE OR WIN32)
  list(REMOVE_ITEM kernel_cu "fusion/gpu/fusion_group_kernel.cu")
  list(REMOVE_ITEM kernel_cu "sparse/gpu/conv_kernel_igemm.cu")
endif()

# New Op only supported by CUDA>=12.0 and Linux
if(((WITH_GPU) AND (CUDA_VERSION VERSION_LESS 12.0))
   OR APPLE
   OR WIN32
   OR WITH_ROCM)
  list(
    REMOVE_ITEM
    kernel_cu
    "legacy/gpu/moe_ops_partial_nosoftmaxtopk_kernel.cu"
    "legacy/gpu/moe_ops_partial_nosoftmaxtopk_grad_kernel.cu"
    "legacy/gpu/moe_gate_dispatch_permute_grad_kernel.cu"
    "legacy/gpu/moe_gate_dispatch_permute_kernel.cu"
    "legacy/gpu/moe_gate_dispatch_and_quant_kernel.cu"
    "legacy/gpu/expand_modality_expert_id_kernel.cu"
    "legacy/gpu/moe_combine_kernel.cu"
    "legacy/gpu/moe_combine_grad_kernel.cu"
    "legacy/gpu/moe_combine_no_weight_kernel.cu"
    "legacy/gpu/moe_combine_no_weight_grad_kernel.cu"
    "legacy/gpu/cal_aux_loss_kernel.cu"
    "legacy/gpu/cal_aux_loss_grad_kernel.cu"
    "legacy/gpu/ext_build_src_rank_and_local_expert_id_kernel.cu"
    "legacy/gpu/moe_gate_dispatch_kernel.cu"
    "legacy/gpu/moe_gate_dispatch_grad_kernel.cu"
    "legacy/gpu/int_bincount.cu"
    "legacy/gpu/batched_gemm.cu"
    "legacy/gpu/fp8_gemm_blockwise_kernel.cu"
    "legacy/gpu/fp8_gemm_blockwise_kernel.cu"
    "legacy/gpu/fp8_quant_blockwise_kernel.cu"
    "legacy/gpu/fast_layernorm_kernel.cu"
    "legacy/gpu/fast_layernorm_grad_kernel.cu"
    "legacy/gpu/fast_rmsnorm_kernel.cu"
    "legacy/gpu/fast_rmsnorm_grad_kernel.cu"
    "legacy/gpu/ln.cu"
    "legacy/gpu/ln_bwd_semi_cuda_kernel.cu"
    "legacy/gpu/ln_fwd_cuda_kernel.cu"
    "fusion/gpu/fused_act_dequant_kernel.cu"
    "fusion/gpu/fused_stack_transpose_quant_kernel.cu"
    "fusion/gpu/fused_transpose_split_quant_kernel.cu"
    "fusion/gpu/fused_transpose_wlch_split_quant_kernel.cu"
    "fusion/gpu/fused_swiglu_weighted_bwd_kernel.cu"
    "fusion/gpu/fused_weighted_swiglu_act_quant_kernel.cu")
endif()

if(((WITH_GPU) AND (CUDA_VERSION VERSION_LESS 11.8))
   OR APPLE
   OR WITH_ROCM)
  list(REMOVE_ITEM kernel_cu "legacy/gpu/layer_norm_cuda_kernel.cu")
endif()

if(WIN32
   OR WITH_ROCM
   OR ((WITH_GPU) AND (CUDA_VERSION VERSION_LESS 12.0)))
  list(REMOVE_ITEM kernel_cu "stride/matmul_grad_stride_kernel.cu")
endif()

# Get flag for CUDA arch >= 80
set(has_arch_ge80 FALSE)
foreach(arch ${NVCC_ARCH_BIN})
  if(${arch} GREATER_EQUAL 80)
    set(has_arch_ge80 TRUE)
    break()
  endif()
endforeach()

if(WITH_GPU
   AND (NOT has_arch_ge80 OR (CUDA_VERSION VERSION_LESS 12.0))
   OR WIN32
   OR APPLE
   OR WITH_ROCM) # need support BF16
  list(REMOVE_ITEM kernel_gpu "gpu/moe_permute_kernel.cu"
       "gpu/moe_unpermute_kernel.cu")
endif()

if(NOT WITH_DGC)
  list(REMOVE_ITEM kernel_gpu "gpu/dgc_kernel.cu")
endif()

if(NOT WITH_MAGMA)
  list(REMOVE_ITEM kernel_gpu "gpu/eig_kernel.cu" "gpu/eig_grad_kernel.cu")
endif()

if(((WITH_GPU) AND (CUDA_VERSION VERSION_LESS 11.8))
   OR APPLE
   OR WITH_ROCM)
  list(REMOVE_ITEM kernel_gpu "gpu/rms_norm_cuda_kernel.cu")
endif()

if(WITH_CUTLASS)
  add_custom_target(
    gemm_epilogue_compile_script ALL
    COMMAND bash compile.sh "${PYTHON_EXECUTABLE}" "${CUDA_TOOLKIT_ROOT_DIR}"
            \"${NVCC_ARCH_BIN}\" "${CMAKE_COMMAND}"
    WORKING_DIRECTORY
      ${PADDLE_SOURCE_DIR}/paddle/phi/kernels/fusion/cutlass/gemm_epilogue
    COMMENT "GemmEpilogue compile script")
  add_custom_target(
    fused_conv2d_add_act_compile_script ALL
    COMMAND bash compile.sh "${PYTHON_EXECUTABLE}" "${CUDA_TOOLKIT_ROOT_DIR}"
            \"${NVCC_ARCH_BIN}\" "${CMAKE_COMMAND}"
    WORKING_DIRECTORY
      ${PADDLE_SOURCE_DIR}/paddle/phi/kernels/fusion/cutlass/conv2d
    COMMENT "FusedConv2dAddAct compile script")

  execute_process(
    COMMAND
      ${PYTHON_EXECUTABLE}
      ${PADDLE_SOURCE_DIR}/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention/generate_kernels.py
      --cuda_arch "${NVCC_ARCH_BIN}" --gen_dir "autogen_tmp"
    RESULT_VARIABLE memory_efficient_attention_gen_res)

  execute_process(
    COMMAND
      ${PYTHON_EXECUTABLE}
      ${PADDLE_SOURCE_DIR}/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention/generate_variable_forward_kernels.py
      --cuda_arch "${NVCC_ARCH_BIN}" --gen_dir "autogen_variable_tmp"
    RESULT_VARIABLE memory_efficient_attention_gen_res)

  if(NOT memory_efficient_attention_gen_res EQUAL 0)
    message(
      FATAL_ERROR
        "The memory efficient attention kernel generation errors with NVCC_ARCH_BIN=${NVCC_ARCH_BIN}"
    )
  endif()

  set(autogen_tmp_dir
      ${PADDLE_SOURCE_DIR}/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention/autogen_tmp
  )
  set(autogen_variable_tmp_dir
      ${PADDLE_SOURCE_DIR}/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention/autogen_variable_tmp
  )
  set(autogen_dir
      ${PADDLE_SOURCE_DIR}/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention/autogen
  )
  set(autogen_variable_dir
      ${PADDLE_SOURCE_DIR}/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention/autogen_variable
  )

  file(GLOB generated_files ${autogen_tmp_dir}/*.h ${autogen_tmp_dir}/impl/*.cu)

  file(GLOB variable_generated_files ${autogen_variable_tmp_dir}/*.h
       ${autogen_variable_tmp_dir}/impl/*.cu)

  if(EXISTS ${autogen_dir})
    foreach(gen_file ${generated_files})
      string(REPLACE "autogen_tmp" "autogen" now_file ${gen_file})
      execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
                              "${gen_file}" "${now_file}")
    endforeach()
    message("copy if different ${autogen_dir}")
  else()
    foreach(gen_file ${generated_files})
      string(REPLACE "autogen_tmp" "autogen" now_file ${gen_file})
      execute_process(COMMAND ${CMAKE_COMMAND} -E copy "${gen_file}"
                              "${now_file}")
    endforeach()
    message("copy ${autogen_dir}")
  endif()

  if(EXISTS ${autogen_variable_dir})
    foreach(gen_file ${variable_generated_files})
      string(REPLACE "autogen_variable_tmp" "autogen_variable" now_file
                     ${gen_file})
      execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
                              "${gen_file}" "${now_file}")
    endforeach()
    message("copy if different ${autogen_variable_dir}")
  else()
    foreach(gen_file ${variable_generated_files})
      string(REPLACE "autogen_variable_tmp" "autogen_variable" now_file
                     ${gen_file})
      execute_process(COMMAND ${CMAKE_COMMAND} -E copy "${gen_file}"
                              "${now_file}")
    endforeach()
    message("copy ${autogen_variable_dir}")
  endif()

  file(
    REMOVE_RECURSE
    ${PADDLE_SOURCE_DIR}/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention/autogen_tmp
  )
  file(
    REMOVE_RECURSE
    ${PADDLE_SOURCE_DIR}/paddle/phi/kernels/fusion/cutlass/memory_efficient_attention/autogen_variable_tmp
  )

  execute_process(
    COMMAND
      ${CMAKE_COMMAND} -E make_directory
      "${CMAKE_CURRENT_SOURCE_DIR}/fusion/cutlass/cutlass_kernels/fpA_intB_gemm/autogen_tmp"
    COMMAND ${PYTHON_EXECUTABLE} generic_mixed_gemm_kernelLauncher.py
            --cuda_arch "${NVCC_ARCH_BIN}"
    WORKING_DIRECTORY
      "${CMAKE_CURRENT_SOURCE_DIR}/fusion/cutlass/cutlass_kernels/fpA_intB_gemm"
  )
  set(fpA_intB_gemm_autogen_tmp_dir
      ${CMAKE_CURRENT_SOURCE_DIR}/fusion/cutlass/cutlass_kernels/fpA_intB_gemm/autogen_tmp
  )
  set(fpA_intB_gemm_autogen_dir
      ${CMAKE_CURRENT_SOURCE_DIR}/fusion/cutlass/cutlass_kernels/fpA_intB_gemm/autogen
  )

  file(GLOB fpA_intB_gemm_autogen_files ${fpA_intB_gemm_autogen_tmp_dir}/*.h
       ${fpA_intB_gemm_autogen_tmp_dir}/*.cu)

  if(EXISTS ${fpA_intB_gemm_autogen_dir})
    foreach(gen_file ${fpA_intB_gemm_autogen_files})
      string(REPLACE "autogen_tmp" "autogen" now_file ${gen_file})
      execute_process(COMMAND ${CMAKE_COMMAND} -E copy_if_different
                              "${gen_file}" "${now_file}")
    endforeach()
    message("copy if different ${fpA_intB_gemm_autogen_dir}")
  else()
    foreach(gen_file ${fpA_intB_gemm_autogen_files})
      string(REPLACE "autogen_tmp" "autogen" now_file ${gen_file})
      execute_process(COMMAND ${CMAKE_COMMAND} -E copy "${gen_file}"
                              "${now_file}")
    endforeach()
    message("copy ${fpA_intB_gemm_autogen_dir}")
  endif()

  file(
    GLOB cutlass_cu
    RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
    "fusion/cutlass/*.cu"
    "fusion/cutlass/memory_efficient_attention/autogen/impl/*.cu"
    "fusion/cutlass/memory_efficient_attention/autogen_variable/impl/*.cu"
    "fusion/cutlass/cutlass_kernels/fpA_intB_gemm/autogen/*.cu"
    "fusion/cutlass/cutlass_kernels/fpA_intB_gemm/*.cu")

  list(APPEND kernel_cu ${cutlass_cu})
endif()

if(NOT WITH_CUDNN_FRONTEND)
  list(
    REMOVE_ITEM
    kernel_cu
    "fusion/gpu/fused_scale_bias_relu_conv_bn_kernel.cu"
    "fusion/gpu/fused_scale_bias_add_relu_kernel.cu"
    "fusion/gpu/fused_dconv_drelu_dbn_kernel.cu"
    "fusion/gpu/fused_dot_product_attention_op.cu"
    "fusion/gpu/max_pool2d_v2_grad_kernel.cu"
    "fusion/gpu/max_pool2d_v2_kernel.cu")
endif()

# Note(qili93): remove kernels not supported on DCU yet
if(WITH_ROCM)
  list(
    REMOVE_ITEM
    kernel_cu
    "legacy/gpu/tensor_debug.cu"
    "gpudnn/mha_cudnn_frontend.cu"
    "fusion/gpu/fused_attention_kernel.cu"
    "fusion/gpu/fused_attention_grad_kernel.cu"
    "fusion/gpu/fused_bn_add_activation_grad_kernel.cu"
    "fusion/gpu/fused_bn_add_activation_kernel.cu"
    "fusion/gpu/fused_feedforward_kernel.cu"
    "fusion/gpu/fused_feedforward_grad_kernel.cu"
    "fusion/gpu/fused_multi_transformer_int8_kernel.cu"
    "fusion/gpu/fused_multi_transformer_kernel.cu"
    "fusion/gpu/fusion_transpose_flatten_concat_kernel.cu")
  list(
    REMOVE_ITEM
    kernel_gpu
    "gpu/affine_grid_grad_kernel.cu"
    "gpu/apply_per_channel_scale_kernel.cu"
    "gpu/calc_reduced_attn_kernel.cu"
    "gpu/eigvalsh_kernel.cu"
    "gpu/lu_kernel.cu"
    "gpu/matrix_rank_kernel.cu"
    "gpu/matrix_rank_tol_kernel.cu"
    "gpu/svd_kernel.cu"
    "gpu/cuda_gemm_kernel.cu")
endif()

# Remove AP kernel when CINN is not enabled.
if(NOT WITH_CINN)
  list(REMOVE_ITEM kernel_cu "gpu/ap_facade_kernel.cu"
       "gpu/ap_trivial_fusion_begin_kernel.cu"
       "gpu/ap_trivial_fusion_end_kernel.cu" "gpu/ap_variadic_kernel.cu")
  list(REMOVE_ITEM kernel_gpu "gpu/ap_facade_kernel.cu"
       "gpu/ap_trivial_fusion_begin_kernel.cu"
       "gpu/ap_trivial_fusion_end_kernel.cu" "gpu/ap_variadic_kernel.cu")
endif()

set(cc_search_pattern
    "*.cc"
    "cpu/*.cc"
    "legacy/*.cc"
    "legacy/cpu/*.cc"
    "selected_rows/*.cc"
    "selected_rows/cpu/*.cc"
    "sparse/*.cc"
    "sparse/cpu/*.cc"
    "legacy/*.cc"
    "legacy/cpu/*.cc"
    "strings/*.cc"
    "strings/cpu/*.cc"
    "fusion/*.cc"
    "stride/*.cc"
    "fusion/cpu/*.cc")

if(WITH_ONEDNN)
  set(cc_search_pattern ${cc_search_pattern} "legacy/onednn/*.cc" "onednn/*.cc"
                        "fusion/onednn/*.cc")
endif()

if(WITH_CUSTOM_DEVICE)
  set(cc_search_pattern ${cc_search_pattern} "custom/*.cc")
endif()

file(
  GLOB kernel_cc
  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
  ${cc_search_pattern})

if(NOT
   (WITH_AVX
    AND AVX512F_FOUND
    AND AVX512F_FLAG
    AND WITH_MKL))
  list(REMOVE_ITEM kernel_cc "fusion/cpu/fused_layer_norm_avx_kernel.cc")
  list(REMOVE_ITEM kernel_cc "fusion/cpu/self_dp_attention_kernel.cc")
  list(REMOVE_ITEM kernel_cc "fusion/cpu/fused_rms_norm_quant_avx_kernel.cc")
endif()

file(
  GLOB kernel_xpu
  RELATIVE "${CMAKE_CURRENT_SOURCE_DIR}"
  "xpu/*.cc" "legacy/xpu/*.cc" "selected_rows/xpu/*.cc" "fusion/xpu/*.cc"
  "sparse/xpu/*.cc")

if(WITH_GPU OR WITH_ROCM)
  set(kernels_gpu_srcs "")
  collect_srcs(kernels_srcs SRCS ${kernel_cu})
  kernel_declare("${kernel_cu}")
  collect_srcs(kernels_gpu_srcs SRCS ${kernel_gpu})
  kernel_declare("${kernel_gpu}")
endif()

if(WITH_XPU)
  if(WITH_XPU_KP)
    file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/kps/
         DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/kps/)
    file(COPY ${CMAKE_CURRENT_SOURCE_DIR}/legacy/kps/
         DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/kps/)
    file(GLOB kernel_xpu_kps "${CMAKE_CURRENT_BINARY_DIR}/kps/*.cu")
    foreach(kernel ${kernel_xpu_kps})
      get_filename_component(name ${kernel} NAME_WE)
      file(RENAME ${kernel} "${CMAKE_CURRENT_BINARY_DIR}/kps/${name}.kps")
    endforeach()
    file(GLOB kernel_xpu_kps "${CMAKE_CURRENT_BINARY_DIR}/kps/*.kps")
    collect_generated_srcs(kernels_srcs SRCS ${kernel_xpu_kps})

    foreach(kernel ${kernel_cc})
      configure_file(${CMAKE_CURRENT_SOURCE_DIR}/${kernel}
                     ${CMAKE_CURRENT_BINARY_DIR}/${kernel} COPYONLY)
    endforeach()
    file(GLOB_RECURSE kernel_xpu_cc "${CMAKE_CURRENT_BINARY_DIR}/*.cc")
    collect_generated_srcs(kernels_srcs SRCS ${kernel_xpu_cc})
    set(kernel_cc "")
  endif()

  # The kernel related to xpufft must have WITH_XPU_FFT enabled.
  if(NOT WITH_XPU_FFT)
    list(
      REMOVE_ITEM
      kernel_xpu
      "xpu/complex_kernel.cc"
      "xpu/complex_grad_kernel.cc"
      "xpu/fft_kernel.cc"
      "xpu/fft_grad_kernel.cc"
      "xpu/as_real_kernel.cc"
      "xpu/as_complex_kernel.cc")
  endif()

  if(NOT WITH_MAGMA)
    list(REMOVE_ITEM kernel_xpu "xpu/eig_kernel.cc")
  endif()

  collect_srcs(kernels_srcs SRCS ${kernel_xpu})
  kernel_declare("${kernel_xpu}")
  kernel_declare("${kernel_xpu_kps}")
  kernel_declare("${kernel_xpu_cc}")
endif()

collect_srcs(kernels_srcs SRCS ${kernel_cc})
kernel_declare("${kernel_cc}")

if(NOT "${KERNEL_LIST}" STREQUAL "")
  prune_declaration_h()
endif()