//===- IntrinsicsAARCH64.td - Defines AARCH64 intrinsics ---*- tablegen -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file defines all of the AARCH64-specific intrinsics.
//
//===----------------------------------------------------------------------===//

let TargetPrefix = "aarch64" in {

def int_aarch64_ldxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty],
                                 [IntrNoFree, IntrWillReturn]>;
def int_aarch64_ldaxr : Intrinsic<[llvm_i64_ty], [llvm_anyptr_ty],
                                  [IntrNoFree, IntrWillReturn]>;
def int_aarch64_stxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty],
                                 [IntrNoFree, IntrWillReturn]>;
def int_aarch64_stlxr : Intrinsic<[llvm_i32_ty], [llvm_i64_ty, llvm_anyptr_ty],
                                  [IntrNoFree, IntrWillReturn]>;

def int_aarch64_ldxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty],
                                 [IntrNoFree, IntrWillReturn]>;
def int_aarch64_ldaxp : Intrinsic<[llvm_i64_ty, llvm_i64_ty], [llvm_ptr_ty],
                                  [IntrNoFree, IntrWillReturn]>;
def int_aarch64_stxp : Intrinsic<[llvm_i32_ty],
                               [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty],
                               [IntrNoFree, IntrWillReturn]>;
def int_aarch64_stlxp : Intrinsic<[llvm_i32_ty],
                                  [llvm_i64_ty, llvm_i64_ty, llvm_ptr_ty],
                                  [IntrNoFree, IntrWillReturn]>;

def int_aarch64_clrex : Intrinsic<[]>;

def int_aarch64_sdiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
                                LLVMMatchType<0>], [IntrNoMem]>;
def int_aarch64_udiv : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>,
                                LLVMMatchType<0>], [IntrNoMem]>;

def int_aarch64_fjcvtzs : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_double_ty], [IntrNoMem]>;

def int_aarch64_cls: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;
def int_aarch64_cls64: DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i64_ty], [IntrNoMem]>;

def int_aarch64_frint32z
    : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ],
                            [ IntrNoMem ]>;
def int_aarch64_frint64z
    : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ],
                            [ IntrNoMem ]>;
def int_aarch64_frint32x
    : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ],
                            [ IntrNoMem ]>;
def int_aarch64_frint64x
    : DefaultAttrsIntrinsic<[ llvm_anyfloat_ty ], [ LLVMMatchType<0> ],
                            [ IntrNoMem ]>;

//===----------------------------------------------------------------------===//
// HINT

def int_aarch64_hint : DefaultAttrsIntrinsic<[], [llvm_i32_ty]>;

def int_aarch64_break : Intrinsic<[], [llvm_i32_ty],
    [IntrNoMem, IntrHasSideEffects, IntrNoReturn, IntrCold, ImmArg<ArgIndex<0>>]>;

//===----------------------------------------------------------------------===//
// Data Barrier Instructions

def int_aarch64_dmb : GCCBuiltin<"__builtin_arm_dmb">, MSBuiltin<"__dmb">,
                      Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;
def int_aarch64_dsb : GCCBuiltin<"__builtin_arm_dsb">, MSBuiltin<"__dsb">,
                      Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;
def int_aarch64_isb : GCCBuiltin<"__builtin_arm_isb">, MSBuiltin<"__isb">,
                      Intrinsic<[], [llvm_i32_ty], [IntrNoFree, IntrWillReturn]>;

// A space-consuming intrinsic primarily for testing block and jump table
// placements. The first argument is the number of bytes this "instruction"
// takes up, the second and return value are essentially chains, used to force
// ordering during ISel.
def int_aarch64_space : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i64_ty], []>;

}

//===----------------------------------------------------------------------===//
// Advanced SIMD (NEON)

let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
  class AdvSIMD_2Scalar_Float_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
                [IntrNoMem]>;

  class AdvSIMD_FPToIntRounding_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty], [IntrNoMem]>;

  class AdvSIMD_1IntArg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  class AdvSIMD_1FloatArg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  class AdvSIMD_1VectorArg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>], [IntrNoMem]>;
  class AdvSIMD_1VectorArg_Expand_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  class AdvSIMD_1VectorArg_Long_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMTruncatedType<0>], [IntrNoMem]>;
  class AdvSIMD_1IntArg_Narrow_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyint_ty], [IntrNoMem]>;
  class AdvSIMD_1VectorArg_Narrow_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMExtendedType<0>], [IntrNoMem]>;
  class AdvSIMD_1VectorArg_Int_Across_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyvector_ty], [IntrNoMem]>;
  class AdvSIMD_1VectorArg_Float_Across_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyvector_ty], [IntrNoMem]>;

  class AdvSIMD_2IntArg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
                [IntrNoMem]>;
  class AdvSIMD_2FloatArg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
                [IntrNoMem]>;
  class AdvSIMD_2VectorArg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMMatchType<0>, LLVMMatchType<0>],
                [IntrNoMem]>;
  class AdvSIMD_2VectorArg_Compare_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMMatchType<1>],
                [IntrNoMem]>;
  class AdvSIMD_2Arg_FloatCompare_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, LLVMMatchType<1>],
                [IntrNoMem]>;
  class AdvSIMD_2VectorArg_Long_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMTruncatedType<0>, LLVMTruncatedType<0>],
                [IntrNoMem]>;
  class AdvSIMD_2VectorArg_Wide_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>, LLVMTruncatedType<0>],
                [IntrNoMem]>;
  class AdvSIMD_2VectorArg_Narrow_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMExtendedType<0>, LLVMExtendedType<0>],
                [IntrNoMem]>;
  class AdvSIMD_2Arg_Scalar_Narrow_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyint_ty],
                [LLVMExtendedType<0>, llvm_i32_ty],
                [IntrNoMem]>;
  class AdvSIMD_2VectorArg_Scalar_Expand_BySize_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [llvm_anyvector_ty],
                [IntrNoMem]>;
  class AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMTruncatedType<0>],
                [IntrNoMem]>;
  class AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMTruncatedType<0>, llvm_i32_ty],
                [IntrNoMem]>;
  class AdvSIMD_2VectorArg_Tied_Narrow_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty],
                [IntrNoMem]>;
  class AdvSIMD_2VectorArg_Lane_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyint_ty],
                [LLVMMatchType<0>, llvm_anyint_ty, llvm_i32_ty],
                [IntrNoMem]>;

  class AdvSIMD_3IntArg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyint_ty],
                [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
                [IntrNoMem]>;
  class AdvSIMD_3VectorArg_Intrinsic
      : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
               [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
               [IntrNoMem]>;
  class AdvSIMD_3VectorArg_Scalar_Intrinsic
      : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
               [LLVMMatchType<0>, LLVMMatchType<0>, llvm_i32_ty],
               [IntrNoMem]>;
  class AdvSIMD_3VectorArg_Tied_Narrow_Intrinsic
      : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
               [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty,
                LLVMMatchType<1>], [IntrNoMem]>;
  class AdvSIMD_3VectorArg_Scalar_Tied_Narrow_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMHalfElementsVectorType<0>, llvm_anyvector_ty, llvm_i32_ty],
                [IntrNoMem]>;
  class AdvSIMD_CvtFxToFP_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyfloat_ty], [llvm_anyint_ty, llvm_i32_ty],
                [IntrNoMem]>;
  class AdvSIMD_CvtFPToFx_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyint_ty], [llvm_anyfloat_ty, llvm_i32_ty],
                [IntrNoMem]>;

  class AdvSIMD_1Arg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_any_ty], [LLVMMatchType<0>], [IntrNoMem]>;

  class AdvSIMD_Dot_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
                [IntrNoMem]>;

  class AdvSIMD_FP16FML_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
                [IntrNoMem]>;

  class AdvSIMD_MatMul_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
                [IntrNoMem]>;

  class AdvSIMD_FML_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>, llvm_anyvector_ty, LLVMMatchType<1>],
                [IntrNoMem]>;

  class AdvSIMD_BF16FML_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_v4f32_ty],
                [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty],
                [IntrNoMem]>;
}

// Arithmetic ops

let TargetPrefix = "aarch64", IntrProperties = [IntrNoMem] in {
  // Vector Add Across Lanes
  def int_aarch64_neon_saddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
  def int_aarch64_neon_uaddv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
  def int_aarch64_neon_faddv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;

  // Vector Long Add Across Lanes
  def int_aarch64_neon_saddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
  def int_aarch64_neon_uaddlv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;

  // Vector Halving Add
  def int_aarch64_neon_shadd : AdvSIMD_2VectorArg_Intrinsic;
  def int_aarch64_neon_uhadd : AdvSIMD_2VectorArg_Intrinsic;

  // Vector Rounding Halving Add
  def int_aarch64_neon_srhadd : AdvSIMD_2VectorArg_Intrinsic;
  def int_aarch64_neon_urhadd : AdvSIMD_2VectorArg_Intrinsic;

  // Vector Saturating Add
  def int_aarch64_neon_sqadd : AdvSIMD_2IntArg_Intrinsic;
  def int_aarch64_neon_suqadd : AdvSIMD_2IntArg_Intrinsic;
  def int_aarch64_neon_usqadd : AdvSIMD_2IntArg_Intrinsic;
  def int_aarch64_neon_uqadd : AdvSIMD_2IntArg_Intrinsic;

  // Vector Add High-Half
  // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
  // header is no longer supported.
  def int_aarch64_neon_addhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;

  // Vector Rounding Add High-Half
  def int_aarch64_neon_raddhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;

  // Vector Saturating Doubling Multiply High
  def int_aarch64_neon_sqdmulh : AdvSIMD_2IntArg_Intrinsic;
  def int_aarch64_neon_sqdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;
  def int_aarch64_neon_sqdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;

  // Vector Saturating Rounding Doubling Multiply High
  def int_aarch64_neon_sqrdmulh : AdvSIMD_2IntArg_Intrinsic;
  def int_aarch64_neon_sqrdmulh_lane : AdvSIMD_2VectorArg_Lane_Intrinsic;
  def int_aarch64_neon_sqrdmulh_laneq : AdvSIMD_2VectorArg_Lane_Intrinsic;

  def int_aarch64_neon_sqrdmlah : AdvSIMD_3IntArg_Intrinsic;
  def int_aarch64_neon_sqrdmlsh : AdvSIMD_3IntArg_Intrinsic;

  // Vector Polynominal Multiply
  def int_aarch64_neon_pmul : AdvSIMD_2VectorArg_Intrinsic;

  // Vector Long Multiply
  def int_aarch64_neon_smull : AdvSIMD_2VectorArg_Long_Intrinsic;
  def int_aarch64_neon_umull : AdvSIMD_2VectorArg_Long_Intrinsic;
  def int_aarch64_neon_pmull : AdvSIMD_2VectorArg_Long_Intrinsic;

  // 64-bit polynomial multiply really returns an i128, which is not legal. Fake
  // it with a v16i8.
  def int_aarch64_neon_pmull64 :
        DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_i64_ty, llvm_i64_ty], [IntrNoMem]>;

  // Vector Extending Multiply
  def int_aarch64_neon_fmulx : AdvSIMD_2FloatArg_Intrinsic {
    let IntrProperties = [IntrNoMem, Commutative];
  }

  // Vector Saturating Doubling Long Multiply
  def int_aarch64_neon_sqdmull : AdvSIMD_2VectorArg_Long_Intrinsic;
  def int_aarch64_neon_sqdmulls_scalar
    : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_i32_ty, llvm_i32_ty], [IntrNoMem]>;

  // Vector Halving Subtract
  def int_aarch64_neon_shsub : AdvSIMD_2VectorArg_Intrinsic;
  def int_aarch64_neon_uhsub : AdvSIMD_2VectorArg_Intrinsic;

  // Vector Saturating Subtract
  def int_aarch64_neon_sqsub : AdvSIMD_2IntArg_Intrinsic;
  def int_aarch64_neon_uqsub : AdvSIMD_2IntArg_Intrinsic;

  // Vector Subtract High-Half
  // FIXME: this is a legacy intrinsic for aarch64_simd.h. Remove it when that
  // header is no longer supported.
  def int_aarch64_neon_subhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;

  // Vector Rounding Subtract High-Half
  def int_aarch64_neon_rsubhn : AdvSIMD_2VectorArg_Narrow_Intrinsic;

  // Vector Compare Absolute Greater-than-or-equal
  def int_aarch64_neon_facge : AdvSIMD_2Arg_FloatCompare_Intrinsic;

  // Vector Compare Absolute Greater-than
  def int_aarch64_neon_facgt : AdvSIMD_2Arg_FloatCompare_Intrinsic;

  // Vector Absolute Difference
  def int_aarch64_neon_sabd : AdvSIMD_2VectorArg_Intrinsic;
  def int_aarch64_neon_uabd : AdvSIMD_2VectorArg_Intrinsic;
  def int_aarch64_neon_fabd : AdvSIMD_2VectorArg_Intrinsic;

  // Scalar Absolute Difference
  def int_aarch64_sisd_fabd : AdvSIMD_2Scalar_Float_Intrinsic;

  // Vector Max
  def int_aarch64_neon_smax : AdvSIMD_2VectorArg_Intrinsic;
  def int_aarch64_neon_umax : AdvSIMD_2VectorArg_Intrinsic;
  def int_aarch64_neon_fmax : AdvSIMD_2FloatArg_Intrinsic;
  def int_aarch64_neon_fmaxnmp : AdvSIMD_2VectorArg_Intrinsic;

  // Vector Max Across Lanes
  def int_aarch64_neon_smaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
  def int_aarch64_neon_umaxv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
  def int_aarch64_neon_fmaxv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
  def int_aarch64_neon_fmaxnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;

  // Vector Min
  def int_aarch64_neon_smin : AdvSIMD_2VectorArg_Intrinsic;
  def int_aarch64_neon_umin : AdvSIMD_2VectorArg_Intrinsic;
  def int_aarch64_neon_fmin : AdvSIMD_2FloatArg_Intrinsic;
  def int_aarch64_neon_fminnmp : AdvSIMD_2VectorArg_Intrinsic;

  // Vector Min/Max Number
  def int_aarch64_neon_fminnm : AdvSIMD_2FloatArg_Intrinsic;
  def int_aarch64_neon_fmaxnm : AdvSIMD_2FloatArg_Intrinsic;

  // Vector Min Across Lanes
  def int_aarch64_neon_sminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
  def int_aarch64_neon_uminv : AdvSIMD_1VectorArg_Int_Across_Intrinsic;
  def int_aarch64_neon_fminv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;
  def int_aarch64_neon_fminnmv : AdvSIMD_1VectorArg_Float_Across_Intrinsic;

  // Pairwise Add
  def int_aarch64_neon_addp : AdvSIMD_2VectorArg_Intrinsic;
  def int_aarch64_neon_faddp : AdvSIMD_2VectorArg_Intrinsic;

  // Long Pairwise Add
  // FIXME: In theory, we shouldn't need intrinsics for saddlp or
  // uaddlp, but tblgen's type inference currently can't handle the
  // pattern fragments this ends up generating.
  def int_aarch64_neon_saddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;
  def int_aarch64_neon_uaddlp : AdvSIMD_1VectorArg_Expand_Intrinsic;

  // Folding Maximum
  def int_aarch64_neon_smaxp : AdvSIMD_2VectorArg_Intrinsic;
  def int_aarch64_neon_umaxp : AdvSIMD_2VectorArg_Intrinsic;
  def int_aarch64_neon_fmaxp : AdvSIMD_2VectorArg_Intrinsic;

  // Folding Minimum
  def int_aarch64_neon_sminp : AdvSIMD_2VectorArg_Intrinsic;
  def int_aarch64_neon_uminp : AdvSIMD_2VectorArg_Intrinsic;
  def int_aarch64_neon_fminp : AdvSIMD_2VectorArg_Intrinsic;

  // Reciprocal Estimate/Step
  def int_aarch64_neon_frecps : AdvSIMD_2FloatArg_Intrinsic;
  def int_aarch64_neon_frsqrts : AdvSIMD_2FloatArg_Intrinsic;

  // Reciprocal Exponent
  def int_aarch64_neon_frecpx : AdvSIMD_1FloatArg_Intrinsic;

  // Vector Saturating Shift Left
  def int_aarch64_neon_sqshl : AdvSIMD_2IntArg_Intrinsic;
  def int_aarch64_neon_uqshl : AdvSIMD_2IntArg_Intrinsic;

  // Vector Rounding Shift Left
  def int_aarch64_neon_srshl : AdvSIMD_2IntArg_Intrinsic;
  def int_aarch64_neon_urshl : AdvSIMD_2IntArg_Intrinsic;

  // Vector Saturating Rounding Shift Left
  def int_aarch64_neon_sqrshl : AdvSIMD_2IntArg_Intrinsic;
  def int_aarch64_neon_uqrshl : AdvSIMD_2IntArg_Intrinsic;

  // Vector Signed->Unsigned Shift Left by Constant
  def int_aarch64_neon_sqshlu : AdvSIMD_2IntArg_Intrinsic;

  // Vector Signed->Unsigned Narrowing Saturating Shift Right by Constant
  def int_aarch64_neon_sqshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;

  // Vector Signed->Unsigned Rounding Narrowing Saturating Shift Right by Const
  def int_aarch64_neon_sqrshrun : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;

  // Vector Narrowing Shift Right by Constant
  def int_aarch64_neon_sqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
  def int_aarch64_neon_uqshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;

  // Vector Rounding Narrowing Shift Right by Constant
  def int_aarch64_neon_rshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;

  // Vector Rounding Narrowing Saturating Shift Right by Constant
  def int_aarch64_neon_sqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;
  def int_aarch64_neon_uqrshrn : AdvSIMD_2Arg_Scalar_Narrow_Intrinsic;

  // Vector Shift Left
  def int_aarch64_neon_sshl : AdvSIMD_2IntArg_Intrinsic;
  def int_aarch64_neon_ushl : AdvSIMD_2IntArg_Intrinsic;

  // Vector Widening Shift Left by Constant
  def int_aarch64_neon_shll : AdvSIMD_2VectorArg_Scalar_Wide_BySize_Intrinsic;
  def int_aarch64_neon_sshll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;
  def int_aarch64_neon_ushll : AdvSIMD_2VectorArg_Scalar_Wide_Intrinsic;

  // Vector Shift Right by Constant and Insert
  def int_aarch64_neon_vsri : AdvSIMD_3VectorArg_Scalar_Intrinsic;

  // Vector Shift Left by Constant and Insert
  def int_aarch64_neon_vsli : AdvSIMD_3VectorArg_Scalar_Intrinsic;

  // Vector Saturating Narrow
  def int_aarch64_neon_scalar_sqxtn: AdvSIMD_1IntArg_Narrow_Intrinsic;
  def int_aarch64_neon_scalar_uqxtn : AdvSIMD_1IntArg_Narrow_Intrinsic;
  def int_aarch64_neon_sqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;
  def int_aarch64_neon_uqxtn : AdvSIMD_1VectorArg_Narrow_Intrinsic;

  // Vector Saturating Extract and Unsigned Narrow
  def int_aarch64_neon_scalar_sqxtun : AdvSIMD_1IntArg_Narrow_Intrinsic;
  def int_aarch64_neon_sqxtun : AdvSIMD_1VectorArg_Narrow_Intrinsic;

  // Vector Absolute Value
  def int_aarch64_neon_abs : AdvSIMD_1Arg_Intrinsic;

  // Vector Saturating Absolute Value
  def int_aarch64_neon_sqabs : AdvSIMD_1IntArg_Intrinsic;

  // Vector Saturating Negation
  def int_aarch64_neon_sqneg : AdvSIMD_1IntArg_Intrinsic;

  // Vector Count Leading Sign Bits
  def int_aarch64_neon_cls : AdvSIMD_1VectorArg_Intrinsic;

  // Vector Reciprocal Estimate
  def int_aarch64_neon_urecpe : AdvSIMD_1VectorArg_Intrinsic;
  def int_aarch64_neon_frecpe : AdvSIMD_1FloatArg_Intrinsic;

  // Vector Square Root Estimate
  def int_aarch64_neon_ursqrte : AdvSIMD_1VectorArg_Intrinsic;
  def int_aarch64_neon_frsqrte : AdvSIMD_1FloatArg_Intrinsic;

  // Vector Conversions Between Half-Precision and Single-Precision.
  def int_aarch64_neon_vcvtfp2hf
    : DefaultAttrsIntrinsic<[llvm_v4i16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
  def int_aarch64_neon_vcvthf2fp
    : DefaultAttrsIntrinsic<[llvm_v4f32_ty], [llvm_v4i16_ty], [IntrNoMem]>;

  // Vector Conversions Between Floating-point and Fixed-point.
  def int_aarch64_neon_vcvtfp2fxs : AdvSIMD_CvtFPToFx_Intrinsic;
  def int_aarch64_neon_vcvtfp2fxu : AdvSIMD_CvtFPToFx_Intrinsic;
  def int_aarch64_neon_vcvtfxs2fp : AdvSIMD_CvtFxToFP_Intrinsic;
  def int_aarch64_neon_vcvtfxu2fp : AdvSIMD_CvtFxToFP_Intrinsic;

  // Vector FP->Int Conversions
  def int_aarch64_neon_fcvtas : AdvSIMD_FPToIntRounding_Intrinsic;
  def int_aarch64_neon_fcvtau : AdvSIMD_FPToIntRounding_Intrinsic;
  def int_aarch64_neon_fcvtms : AdvSIMD_FPToIntRounding_Intrinsic;
  def int_aarch64_neon_fcvtmu : AdvSIMD_FPToIntRounding_Intrinsic;
  def int_aarch64_neon_fcvtns : AdvSIMD_FPToIntRounding_Intrinsic;
  def int_aarch64_neon_fcvtnu : AdvSIMD_FPToIntRounding_Intrinsic;
  def int_aarch64_neon_fcvtps : AdvSIMD_FPToIntRounding_Intrinsic;
  def int_aarch64_neon_fcvtpu : AdvSIMD_FPToIntRounding_Intrinsic;
  def int_aarch64_neon_fcvtzs : AdvSIMD_FPToIntRounding_Intrinsic;
  def int_aarch64_neon_fcvtzu : AdvSIMD_FPToIntRounding_Intrinsic;

  // v8.5-A Vector FP Rounding
  def int_aarch64_neon_frint32x : AdvSIMD_1FloatArg_Intrinsic;
  def int_aarch64_neon_frint32z : AdvSIMD_1FloatArg_Intrinsic;
  def int_aarch64_neon_frint64x : AdvSIMD_1FloatArg_Intrinsic;
  def int_aarch64_neon_frint64z : AdvSIMD_1FloatArg_Intrinsic;

  // Scalar FP->Int conversions

  // Vector FP Inexact Narrowing
  def int_aarch64_neon_fcvtxn : AdvSIMD_1VectorArg_Expand_Intrinsic;

  // Scalar FP Inexact Narrowing
  def int_aarch64_sisd_fcvtxn : DefaultAttrsIntrinsic<[llvm_float_ty], [llvm_double_ty],
                                        [IntrNoMem]>;

  // v8.2-A Dot Product
  def int_aarch64_neon_udot : AdvSIMD_Dot_Intrinsic;
  def int_aarch64_neon_sdot : AdvSIMD_Dot_Intrinsic;

  // v8.6-A Matrix Multiply Intrinsics
  def int_aarch64_neon_ummla : AdvSIMD_MatMul_Intrinsic;
  def int_aarch64_neon_smmla : AdvSIMD_MatMul_Intrinsic;
  def int_aarch64_neon_usmmla : AdvSIMD_MatMul_Intrinsic;
  def int_aarch64_neon_usdot : AdvSIMD_Dot_Intrinsic;
  def int_aarch64_neon_bfdot : AdvSIMD_Dot_Intrinsic;
  def int_aarch64_neon_bfmmla
    : DefaultAttrsIntrinsic<[llvm_v4f32_ty],
                [llvm_v4f32_ty, llvm_v8bf16_ty, llvm_v8bf16_ty],
                [IntrNoMem]>;
  def int_aarch64_neon_bfmlalb : AdvSIMD_BF16FML_Intrinsic;
  def int_aarch64_neon_bfmlalt : AdvSIMD_BF16FML_Intrinsic;


  // v8.6-A Bfloat Intrinsics
  def int_aarch64_neon_bfcvt
    : DefaultAttrsIntrinsic<[llvm_bfloat_ty], [llvm_float_ty], [IntrNoMem]>;
  def int_aarch64_neon_bfcvtn
    : DefaultAttrsIntrinsic<[llvm_v8bf16_ty], [llvm_v4f32_ty], [IntrNoMem]>;
  def int_aarch64_neon_bfcvtn2
    : DefaultAttrsIntrinsic<[llvm_v8bf16_ty],
                [llvm_v8bf16_ty, llvm_v4f32_ty],
                [IntrNoMem]>;

  // v8.2-A FP16 Fused Multiply-Add Long
  def int_aarch64_neon_fmlal : AdvSIMD_FP16FML_Intrinsic;
  def int_aarch64_neon_fmlsl : AdvSIMD_FP16FML_Intrinsic;
  def int_aarch64_neon_fmlal2 : AdvSIMD_FP16FML_Intrinsic;
  def int_aarch64_neon_fmlsl2 : AdvSIMD_FP16FML_Intrinsic;

  // v8.3-A Floating-point complex add
  def int_aarch64_neon_vcadd_rot90  : AdvSIMD_2VectorArg_Intrinsic;
  def int_aarch64_neon_vcadd_rot270 : AdvSIMD_2VectorArg_Intrinsic;

  def int_aarch64_neon_vcmla_rot0   : AdvSIMD_3VectorArg_Intrinsic;
  def int_aarch64_neon_vcmla_rot90  : AdvSIMD_3VectorArg_Intrinsic;
  def int_aarch64_neon_vcmla_rot180 : AdvSIMD_3VectorArg_Intrinsic;
  def int_aarch64_neon_vcmla_rot270 : AdvSIMD_3VectorArg_Intrinsic;
}

let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
  class AdvSIMD_2Vector2Index_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [llvm_anyvector_ty, llvm_i64_ty, LLVMMatchType<0>, llvm_i64_ty],
                [IntrNoMem]>;
}

// Vector element to element moves
def int_aarch64_neon_vcopy_lane: AdvSIMD_2Vector2Index_Intrinsic;

let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
  class AdvSIMD_1Vec_Load_Intrinsic
      : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMAnyPointerType<LLVMMatchType<0>>],
                  [IntrReadMem, IntrArgMemOnly]>;
  class AdvSIMD_1Vec_Store_Lane_Intrinsic
    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_i64_ty, llvm_anyptr_ty],
                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;

  class AdvSIMD_2Vec_Load_Intrinsic
    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, llvm_anyvector_ty],
                [LLVMAnyPointerType<LLVMMatchType<0>>],
                [IntrReadMem, IntrArgMemOnly]>;
  class AdvSIMD_2Vec_Load_Lane_Intrinsic
    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>],
                [LLVMMatchType<0>, llvm_anyvector_ty,
                 llvm_i64_ty, llvm_anyptr_ty],
                [IntrReadMem, IntrArgMemOnly]>;
  class AdvSIMD_2Vec_Store_Intrinsic
    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
                     LLVMAnyPointerType<LLVMMatchType<0>>],
                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;
  class AdvSIMD_2Vec_Store_Lane_Intrinsic
    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
                 llvm_i64_ty, llvm_anyptr_ty],
                [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;

  class AdvSIMD_3Vec_Load_Intrinsic
    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty],
                [LLVMAnyPointerType<LLVMMatchType<0>>],
                [IntrReadMem, IntrArgMemOnly]>;
  class AdvSIMD_3Vec_Load_Lane_Intrinsic
    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
                [LLVMMatchType<0>, LLVMMatchType<0>, llvm_anyvector_ty,
                 llvm_i64_ty, llvm_anyptr_ty],
                [IntrReadMem, IntrArgMemOnly]>;
  class AdvSIMD_3Vec_Store_Intrinsic
    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
                     LLVMMatchType<0>, LLVMAnyPointerType<LLVMMatchType<0>>],
                [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;
  class AdvSIMD_3Vec_Store_Lane_Intrinsic
    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty,
                 LLVMMatchType<0>, LLVMMatchType<0>,
                 llvm_i64_ty, llvm_anyptr_ty],
                [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;

  class AdvSIMD_4Vec_Load_Intrinsic
    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
                 LLVMMatchType<0>, llvm_anyvector_ty],
                [LLVMAnyPointerType<LLVMMatchType<0>>],
                [IntrReadMem, IntrArgMemOnly]>;
  class AdvSIMD_4Vec_Load_Lane_Intrinsic
    : DefaultAttrsIntrinsic<[LLVMMatchType<0>, LLVMMatchType<0>,
                 LLVMMatchType<0>, LLVMMatchType<0>],
                [LLVMMatchType<0>, LLVMMatchType<0>,
                 LLVMMatchType<0>, llvm_anyvector_ty,
                 llvm_i64_ty, llvm_anyptr_ty],
                [IntrReadMem, IntrArgMemOnly]>;
  class AdvSIMD_4Vec_Store_Intrinsic
    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
                 LLVMMatchType<0>, LLVMMatchType<0>,
                 LLVMAnyPointerType<LLVMMatchType<0>>],
                [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;
  class AdvSIMD_4Vec_Store_Lane_Intrinsic
    : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, LLVMMatchType<0>,
                 LLVMMatchType<0>, LLVMMatchType<0>,
                 llvm_i64_ty, llvm_anyptr_ty],
                [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;
}

// Memory ops

def int_aarch64_neon_ld1x2 : AdvSIMD_2Vec_Load_Intrinsic;
def int_aarch64_neon_ld1x3 : AdvSIMD_3Vec_Load_Intrinsic;
def int_aarch64_neon_ld1x4 : AdvSIMD_4Vec_Load_Intrinsic;

def int_aarch64_neon_st1x2 : AdvSIMD_2Vec_Store_Intrinsic;
def int_aarch64_neon_st1x3 : AdvSIMD_3Vec_Store_Intrinsic;
def int_aarch64_neon_st1x4 : AdvSIMD_4Vec_Store_Intrinsic;

def int_aarch64_neon_ld2 : AdvSIMD_2Vec_Load_Intrinsic;
def int_aarch64_neon_ld3 : AdvSIMD_3Vec_Load_Intrinsic;
def int_aarch64_neon_ld4 : AdvSIMD_4Vec_Load_Intrinsic;

def int_aarch64_neon_ld2lane : AdvSIMD_2Vec_Load_Lane_Intrinsic;
def int_aarch64_neon_ld3lane : AdvSIMD_3Vec_Load_Lane_Intrinsic;
def int_aarch64_neon_ld4lane : AdvSIMD_4Vec_Load_Lane_Intrinsic;

def int_aarch64_neon_ld2r : AdvSIMD_2Vec_Load_Intrinsic;
def int_aarch64_neon_ld3r : AdvSIMD_3Vec_Load_Intrinsic;
def int_aarch64_neon_ld4r : AdvSIMD_4Vec_Load_Intrinsic;

def int_aarch64_neon_st2  : AdvSIMD_2Vec_Store_Intrinsic;
def int_aarch64_neon_st3  : AdvSIMD_3Vec_Store_Intrinsic;
def int_aarch64_neon_st4  : AdvSIMD_4Vec_Store_Intrinsic;

def int_aarch64_neon_st2lane  : AdvSIMD_2Vec_Store_Lane_Intrinsic;
def int_aarch64_neon_st3lane  : AdvSIMD_3Vec_Store_Lane_Intrinsic;
def int_aarch64_neon_st4lane  : AdvSIMD_4Vec_Store_Lane_Intrinsic;

let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".
  class AdvSIMD_Tbl1_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_v16i8_ty, LLVMMatchType<0>],
                [IntrNoMem]>;
  class AdvSIMD_Tbl2_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>], [IntrNoMem]>;
  class AdvSIMD_Tbl3_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
                 LLVMMatchType<0>],
                [IntrNoMem]>;
  class AdvSIMD_Tbl4_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty, llvm_v16i8_ty,
                 LLVMMatchType<0>],
                [IntrNoMem]>;

  class AdvSIMD_Tbx1_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>, llvm_v16i8_ty, LLVMMatchType<0>],
                [IntrNoMem]>;
  class AdvSIMD_Tbx2_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
                 LLVMMatchType<0>],
                [IntrNoMem]>;
  class AdvSIMD_Tbx3_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
                 llvm_v16i8_ty, LLVMMatchType<0>],
                [IntrNoMem]>;
  class AdvSIMD_Tbx4_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>, llvm_v16i8_ty, llvm_v16i8_ty,
                 llvm_v16i8_ty, llvm_v16i8_ty, LLVMMatchType<0>],
                [IntrNoMem]>;
}
def int_aarch64_neon_tbl1 : AdvSIMD_Tbl1_Intrinsic;
def int_aarch64_neon_tbl2 : AdvSIMD_Tbl2_Intrinsic;
def int_aarch64_neon_tbl3 : AdvSIMD_Tbl3_Intrinsic;
def int_aarch64_neon_tbl4 : AdvSIMD_Tbl4_Intrinsic;

def int_aarch64_neon_tbx1 : AdvSIMD_Tbx1_Intrinsic;
def int_aarch64_neon_tbx2 : AdvSIMD_Tbx2_Intrinsic;
def int_aarch64_neon_tbx3 : AdvSIMD_Tbx3_Intrinsic;
def int_aarch64_neon_tbx4 : AdvSIMD_Tbx4_Intrinsic;

let TargetPrefix = "aarch64" in {
  class FPCR_Get_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem, IntrHasSideEffects]>;
  class FPCR_Set_Intrinsic
    : DefaultAttrsIntrinsic<[], [llvm_i64_ty], [IntrNoMem, IntrHasSideEffects]>;
  class RNDR_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_i64_ty, llvm_i1_ty], [], [IntrNoMem, IntrHasSideEffects]>;
}

// FPCR
def int_aarch64_get_fpcr : FPCR_Get_Intrinsic;
def int_aarch64_set_fpcr : FPCR_Set_Intrinsic;

// Armv8.5-A Random number generation intrinsics
def int_aarch64_rndr : RNDR_Intrinsic;
def int_aarch64_rndrrs : RNDR_Intrinsic;

let TargetPrefix = "aarch64" in {
  class Crypto_AES_DataKey_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty, llvm_v16i8_ty], [IntrNoMem]>;

  class Crypto_AES_Data_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_v16i8_ty], [llvm_v16i8_ty], [IntrNoMem]>;

  // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
  // (v4i32).
  class Crypto_SHA_5Hash4Schedule_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_i32_ty, llvm_v4i32_ty],
                [IntrNoMem]>;

  // SHA intrinsic taking 5 words of the hash (v4i32, i32) and 4 of the schedule
  // (v4i32).
  class Crypto_SHA_1Hash_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty], [IntrNoMem]>;

  // SHA intrinsic taking 8 words of the schedule
  class Crypto_SHA_8Schedule_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;

  // SHA intrinsic taking 12 words of the schedule
  class Crypto_SHA_12Schedule_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
                [IntrNoMem]>;

  // SHA intrinsic taking 8 words of the hash and 4 of the schedule.
  class Crypto_SHA_8Hash4Schedule_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
                [IntrNoMem]>;

  // SHA512 intrinsic taking 2 arguments
  class Crypto_SHA512_2Arg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty], [IntrNoMem]>;

  // SHA512 intrinsic taking 3 Arguments
  class Crypto_SHA512_3Arg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_v2i64_ty],
                [IntrNoMem]>;

  // SHA3 Intrinsics taking 3 arguments
  class Crypto_SHA3_3Arg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
               [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
               [IntrNoMem]>;

  // SHA3 Intrinsic taking 2 arguments
  class Crypto_SHA3_2Arg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty],
               [IntrNoMem]>;

  // SHA3 Intrinsic taking 3 Arguments 1 immediate
  class Crypto_SHA3_2ArgImm_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_v2i64_ty], [llvm_v2i64_ty, llvm_v2i64_ty, llvm_i64_ty],
               [IntrNoMem, ImmArg<ArgIndex<2>>]>;

  class Crypto_SM3_3Vector_Intrinsic
    : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty],
                [IntrNoMem]>;

  class Crypto_SM3_3VectorIndexed_Intrinsic
    : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty, llvm_v4i32_ty, llvm_i64_ty],
                [IntrNoMem, ImmArg<ArgIndex<3>>]>;

  class Crypto_SM4_2Vector_Intrinsic
    : Intrinsic<[llvm_v4i32_ty], [llvm_v4i32_ty, llvm_v4i32_ty], [IntrNoMem]>;
}

// AES
def int_aarch64_crypto_aese   : Crypto_AES_DataKey_Intrinsic;
def int_aarch64_crypto_aesd   : Crypto_AES_DataKey_Intrinsic;
def int_aarch64_crypto_aesmc  : Crypto_AES_Data_Intrinsic;
def int_aarch64_crypto_aesimc : Crypto_AES_Data_Intrinsic;

// SHA1
def int_aarch64_crypto_sha1c  : Crypto_SHA_5Hash4Schedule_Intrinsic;
def int_aarch64_crypto_sha1p  : Crypto_SHA_5Hash4Schedule_Intrinsic;
def int_aarch64_crypto_sha1m  : Crypto_SHA_5Hash4Schedule_Intrinsic;
def int_aarch64_crypto_sha1h  : Crypto_SHA_1Hash_Intrinsic;

def int_aarch64_crypto_sha1su0 : Crypto_SHA_12Schedule_Intrinsic;
def int_aarch64_crypto_sha1su1 : Crypto_SHA_8Schedule_Intrinsic;

// SHA256
def int_aarch64_crypto_sha256h   : Crypto_SHA_8Hash4Schedule_Intrinsic;
def int_aarch64_crypto_sha256h2  : Crypto_SHA_8Hash4Schedule_Intrinsic;
def int_aarch64_crypto_sha256su0 : Crypto_SHA_8Schedule_Intrinsic;
def int_aarch64_crypto_sha256su1 : Crypto_SHA_12Schedule_Intrinsic;

//SHA3
def int_aarch64_crypto_eor3s : Crypto_SHA3_3Arg_Intrinsic;
def int_aarch64_crypto_eor3u : Crypto_SHA3_3Arg_Intrinsic;
def int_aarch64_crypto_bcaxs : Crypto_SHA3_3Arg_Intrinsic;
def int_aarch64_crypto_bcaxu : Crypto_SHA3_3Arg_Intrinsic;
def int_aarch64_crypto_rax1 : Crypto_SHA3_2Arg_Intrinsic;
def int_aarch64_crypto_xar : Crypto_SHA3_2ArgImm_Intrinsic;

// SHA512
def int_aarch64_crypto_sha512h : Crypto_SHA512_3Arg_Intrinsic;
def int_aarch64_crypto_sha512h2 : Crypto_SHA512_3Arg_Intrinsic;
def int_aarch64_crypto_sha512su0 : Crypto_SHA512_2Arg_Intrinsic;
def int_aarch64_crypto_sha512su1 : Crypto_SHA512_3Arg_Intrinsic;

//SM3 & SM4
def int_aarch64_crypto_sm3partw1 : Crypto_SM3_3Vector_Intrinsic;
def int_aarch64_crypto_sm3partw2 : Crypto_SM3_3Vector_Intrinsic;
def int_aarch64_crypto_sm3ss1    : Crypto_SM3_3Vector_Intrinsic;
def int_aarch64_crypto_sm3tt1a   : Crypto_SM3_3VectorIndexed_Intrinsic;
def int_aarch64_crypto_sm3tt1b   : Crypto_SM3_3VectorIndexed_Intrinsic;
def int_aarch64_crypto_sm3tt2a   : Crypto_SM3_3VectorIndexed_Intrinsic;
def int_aarch64_crypto_sm3tt2b   : Crypto_SM3_3VectorIndexed_Intrinsic;
def int_aarch64_crypto_sm4e      : Crypto_SM4_2Vector_Intrinsic;
def int_aarch64_crypto_sm4ekey   : Crypto_SM4_2Vector_Intrinsic;

//===----------------------------------------------------------------------===//
// CRC32

let TargetPrefix = "aarch64" in {

def int_aarch64_crc32b  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
    [IntrNoMem]>;
def int_aarch64_crc32cb : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
    [IntrNoMem]>;
def int_aarch64_crc32h  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
    [IntrNoMem]>;
def int_aarch64_crc32ch : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
    [IntrNoMem]>;
def int_aarch64_crc32w  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
    [IntrNoMem]>;
def int_aarch64_crc32cw : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i32_ty],
    [IntrNoMem]>;
def int_aarch64_crc32x  : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
    [IntrNoMem]>;
def int_aarch64_crc32cx : DefaultAttrsIntrinsic<[llvm_i32_ty], [llvm_i32_ty, llvm_i64_ty],
    [IntrNoMem]>;
}

//===----------------------------------------------------------------------===//
// Memory Tagging Extensions (MTE) Intrinsics
let TargetPrefix = "aarch64" in {
def int_aarch64_irg   : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
    [IntrNoMem, IntrHasSideEffects]>;
def int_aarch64_addg  : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i64_ty],
    [IntrNoMem]>;
def int_aarch64_gmi   : DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_i64_ty],
    [IntrNoMem]>;
def int_aarch64_ldg   : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_ptr_ty],
    [IntrReadMem]>;
def int_aarch64_stg   : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_ptr_ty],
    [IntrWriteMem]>;
def int_aarch64_subp :  DefaultAttrsIntrinsic<[llvm_i64_ty], [llvm_ptr_ty, llvm_ptr_ty],
    [IntrNoMem]>;

// The following are codegen-only intrinsics for stack instrumentation.

// Generate a randomly tagged stack base pointer.
def int_aarch64_irg_sp   : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_i64_ty],
    [IntrNoMem, IntrHasSideEffects]>;

// Transfer pointer tag with offset.
// ptr1 = tagp(ptr0, baseptr, tag_offset) returns a pointer where
// * address is the address in ptr0
// * tag is a function of (tag in baseptr, tag_offset).
// ** Beware, this is not the same function as implemented by the ADDG instruction!
//    Backend optimizations may change tag_offset; the only guarantee is that calls
//    to tagp with the same pair of (baseptr, tag_offset) will produce pointers
//    with the same tag value, assuming the set of excluded tags has not changed.
// Address bits in baseptr and tag bits in ptr0 are ignored.
// When offset between ptr0 and baseptr is a compile time constant, this can be emitted as
//   ADDG ptr1, baseptr, (ptr0 - baseptr), tag_offset
// It is intended that ptr0 is an alloca address, and baseptr is the direct output of llvm.aarch64.irg.sp.
def int_aarch64_tagp : DefaultAttrsIntrinsic<[llvm_anyptr_ty], [LLVMMatchType<0>, llvm_ptr_ty, llvm_i64_ty],
    [IntrNoMem, ImmArg<ArgIndex<2>>]>;

// Update allocation tags for the memory range to match the tag in the pointer argument.
def int_aarch64_settag  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;

// Update allocation tags for the memory range to match the tag in the pointer argument,
// and set memory contents to zero.
def int_aarch64_settag_zero  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty],
    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;

// Update allocation tags for 16-aligned, 16-sized memory region, and store a pair 8-byte values.
def int_aarch64_stgp  : DefaultAttrsIntrinsic<[], [llvm_ptr_ty, llvm_i64_ty, llvm_i64_ty],
    [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
}

//===----------------------------------------------------------------------===//
// Memory Operations (MOPS) Intrinsics
let TargetPrefix = "aarch64" in {
  // Sizes are chosen to correspond to the llvm.memset intrinsic: ptr, i8, i64
  def int_aarch64_mops_memset_tag : DefaultAttrsIntrinsic<[llvm_ptr_ty], [llvm_ptr_ty, llvm_i8_ty, llvm_i64_ty],
      [IntrWriteMem, IntrArgMemOnly, NoCapture<ArgIndex<0>>, WriteOnly<ArgIndex<0>>]>;
}

// Transactional Memory Extension (TME) Intrinsics
let TargetPrefix = "aarch64" in {
def int_aarch64_tstart  : GCCBuiltin<"__builtin_arm_tstart">,
                         Intrinsic<[llvm_i64_ty], [], [IntrWillReturn]>;

def int_aarch64_tcommit : GCCBuiltin<"__builtin_arm_tcommit">, Intrinsic<[], [], [IntrWillReturn]>;

def int_aarch64_tcancel : GCCBuiltin<"__builtin_arm_tcancel">,
                          Intrinsic<[], [llvm_i64_ty], [IntrWillReturn, ImmArg<ArgIndex<0>>]>;

def int_aarch64_ttest   : GCCBuiltin<"__builtin_arm_ttest">,
                          Intrinsic<[llvm_i64_ty], [],
                                    [IntrNoMem, IntrHasSideEffects, IntrWillReturn]>;

// Armv8.7-A load/store 64-byte intrinsics
defvar data512 = !listsplat(llvm_i64_ty, 8);
def int_aarch64_ld64b: Intrinsic<data512, [llvm_ptr_ty]>;
def int_aarch64_st64b: Intrinsic<[], !listconcat([llvm_ptr_ty], data512)>;
def int_aarch64_st64bv: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>;
def int_aarch64_st64bv0: Intrinsic<[llvm_i64_ty], !listconcat([llvm_ptr_ty], data512)>;

}

def llvm_nxv2i1_ty  : LLVMType<nxv2i1>;
def llvm_nxv4i1_ty  : LLVMType<nxv4i1>;
def llvm_nxv8i1_ty  : LLVMType<nxv8i1>;
def llvm_nxv16i1_ty : LLVMType<nxv16i1>;
def llvm_nxv16i8_ty : LLVMType<nxv16i8>;
def llvm_nxv4i32_ty : LLVMType<nxv4i32>;
def llvm_nxv2i64_ty : LLVMType<nxv2i64>;
def llvm_nxv8f16_ty : LLVMType<nxv8f16>;
def llvm_nxv8bf16_ty : LLVMType<nxv8bf16>;
def llvm_nxv4f32_ty : LLVMType<nxv4f32>;
def llvm_nxv2f64_ty : LLVMType<nxv2f64>;

let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".

  class AdvSIMD_SVE_Create_2Vector_Tuple
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [llvm_anyvector_ty, LLVMMatchType<1>],
                [IntrReadMem]>;

  class AdvSIMD_SVE_Create_3Vector_Tuple
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>],
                [IntrReadMem]>;

  class AdvSIMD_SVE_Create_4Vector_Tuple
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [llvm_anyvector_ty, LLVMMatchType<1>, LLVMMatchType<1>,
                 LLVMMatchType<1>],
                [IntrReadMem]>;

  class AdvSIMD_SVE_Set_Vector_Tuple
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>, llvm_i32_ty, llvm_anyvector_ty],
                [IntrReadMem, ImmArg<ArgIndex<1>>]>;

  class AdvSIMD_SVE_Get_Vector_Tuple
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, llvm_i32_ty],
                [IntrReadMem, IntrArgMemOnly, ImmArg<ArgIndex<1>>]>;

  class AdvSIMD_ManyVec_PredLoad_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [llvm_anyvector_ty, LLVMPointerToElt<0>],
                [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_1Vec_PredLoad_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMPointerToElt<0>],
                [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_2Vec_PredLoad_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMPointerToElt<0>],
                [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_3Vec_PredLoad_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMPointerToElt<0>],
                [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_4Vec_PredLoad_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
                 LLVMMatchType<0>],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMPointerToElt<0>],
                [IntrReadMem, IntrArgMemOnly]>;

  class AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMPointerToElt<0>],
                [IntrInaccessibleMemOrArgMemOnly]>;

  class AdvSIMD_1Vec_PredStore_Intrinsic
    : DefaultAttrsIntrinsic<[],
                [llvm_anyvector_ty,
                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMPointerToElt<0>],
                [IntrArgMemOnly, NoCapture<ArgIndex<2>>]>;

  class AdvSIMD_2Vec_PredStore_Intrinsic
      : DefaultAttrsIntrinsic<[],
                  [llvm_anyvector_ty, LLVMMatchType<0>,
                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
                  [IntrArgMemOnly, NoCapture<ArgIndex<3>>]>;

  class AdvSIMD_3Vec_PredStore_Intrinsic
      : DefaultAttrsIntrinsic<[],
                  [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
                  [IntrArgMemOnly, NoCapture<ArgIndex<4>>]>;

  class AdvSIMD_4Vec_PredStore_Intrinsic
      : DefaultAttrsIntrinsic<[],
                  [llvm_anyvector_ty, LLVMMatchType<0>, LLVMMatchType<0>,
                   LLVMMatchType<0>,
                   LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, LLVMPointerToElt<0>],
                  [IntrArgMemOnly, NoCapture<ArgIndex<5>>]>;

  class AdvSIMD_SVE_Index_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMVectorElementType<0>,
                 LLVMVectorElementType<0>],
                [IntrNoMem]>;

  class AdvSIMD_Merged1VectorArg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMMatchType<0>],
                [IntrNoMem]>;

  class AdvSIMD_2VectorArgIndexed_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMMatchType<0>,
                 llvm_i32_ty],
                [IntrNoMem, ImmArg<ArgIndex<2>>]>;

  class AdvSIMD_3VectorArgIndexed_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMMatchType<0>,
                 LLVMMatchType<0>,
                 llvm_i32_ty],
                [IntrNoMem, ImmArg<ArgIndex<3>>]>;

  class AdvSIMD_Pred1VectorArg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMMatchType<0>],
                [IntrNoMem]>;

  class AdvSIMD_Pred2VectorArg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMMatchType<0>,
                 LLVMMatchType<0>],
                [IntrNoMem]>;

  class AdvSIMD_Pred3VectorArg_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMMatchType<0>,
                 LLVMMatchType<0>,
                 LLVMMatchType<0>],
                [IntrNoMem]>;

  class AdvSIMD_SVE_Compare_Intrinsic
    : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 llvm_anyvector_ty,
                 LLVMMatchType<0>],
                [IntrNoMem]>;

  class AdvSIMD_SVE_CompareWide_Intrinsic
    : DefaultAttrsIntrinsic<[LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 llvm_anyvector_ty,
                 llvm_nxv2i64_ty],
                [IntrNoMem]>;

  class AdvSIMD_SVE_Saturating_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>],
                [IntrNoMem]>;

  class AdvSIMD_SVE_SaturatingWithPattern_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 llvm_i32_ty,
                 llvm_i32_ty],
                [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;

  class AdvSIMD_SVE_Saturating_N_Intrinsic<LLVMType T>
    : DefaultAttrsIntrinsic<[T],
                [T, llvm_anyvector_ty],
                [IntrNoMem]>;

  class AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<LLVMType T>
    : DefaultAttrsIntrinsic<[T],
                [T, llvm_i32_ty, llvm_i32_ty],
                [IntrNoMem, ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>]>;

  class AdvSIMD_SVE_CNT_Intrinsic
    : DefaultAttrsIntrinsic<[LLVMVectorOfBitcastsToInt<0>],
                [LLVMVectorOfBitcastsToInt<0>,
                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 llvm_anyvector_ty],
                [IntrNoMem]>;

  class AdvSIMD_SVE_ReduceWithInit_Intrinsic
    : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMVectorElementType<0>,
                 llvm_anyvector_ty],
                [IntrNoMem]>;

  class AdvSIMD_SVE_ShiftByImm_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMMatchType<0>,
                 llvm_i32_ty],
                [IntrNoMem, ImmArg<ArgIndex<2>>]>;

  class AdvSIMD_SVE_ShiftWide_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMMatchType<0>,
                 llvm_nxv2i64_ty],
                [IntrNoMem]>;

  class AdvSIMD_SVE_Unpack_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
               [LLVMSubdivide2VectorType<0>],
               [IntrNoMem]>;

  class AdvSIMD_SVE_CADD_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMMatchType<0>,
                 LLVMMatchType<0>,
                 llvm_i32_ty],
                [IntrNoMem, ImmArg<ArgIndex<3>>]>;

  class AdvSIMD_SVE_CMLA_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMMatchType<0>,
                 LLVMMatchType<0>,
                 LLVMMatchType<0>,
                 llvm_i32_ty],
                [IntrNoMem, ImmArg<ArgIndex<4>>]>;

  class AdvSIMD_SVE_CMLA_LANE_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMMatchType<0>,
                 LLVMMatchType<0>,
                 llvm_i32_ty,
                 llvm_i32_ty],
                [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;

  class AdvSIMD_SVE_DUP_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMVectorElementType<0>],
                [IntrNoMem]>;

  class AdvSIMD_SVE_DUP_Unpred_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty], [LLVMVectorElementType<0>],
                [IntrNoMem]>;

  class AdvSIMD_SVE_DUPQ_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 llvm_i64_ty],
                [IntrNoMem]>;

  class AdvSIMD_SVE_EXPA_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMVectorOfBitcastsToInt<0>],
                [IntrNoMem]>;

  class AdvSIMD_SVE_FCVT_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 llvm_anyvector_ty],
                [IntrNoMem]>;

  class AdvSIMD_SVE_FCVTZS_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMVectorOfBitcastsToInt<0>,
                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 llvm_anyvector_ty],
                [IntrNoMem]>;

  class AdvSIMD_SVE_INSR_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMVectorElementType<0>],
                [IntrNoMem]>;

  class AdvSIMD_SVE_PTRUE_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [llvm_i32_ty],
                [IntrNoMem, ImmArg<ArgIndex<0>>]>;

  class AdvSIMD_SVE_PUNPKHI_Intrinsic
    : DefaultAttrsIntrinsic<[LLVMHalfElementsVectorType<0>],
                [llvm_anyvector_ty],
                [IntrNoMem]>;

  class AdvSIMD_SVE_SCALE_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMMatchType<0>,
                 LLVMVectorOfBitcastsToInt<0>],
                [IntrNoMem]>;

  class AdvSIMD_SVE_SCVTF_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 llvm_anyvector_ty],
                [IntrNoMem]>;

  class AdvSIMD_SVE_TSMUL_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMVectorOfBitcastsToInt<0>],
                [IntrNoMem]>;

  class AdvSIMD_SVE_CNTB_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_i64_ty],
                [llvm_i32_ty],
                [IntrNoMem, ImmArg<ArgIndex<0>>]>;

  class AdvSIMD_SVE_CNTP_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_i64_ty],
                [llvm_anyvector_ty, LLVMMatchType<0>],
                [IntrNoMem]>;

  class AdvSIMD_SVE_DOT_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMSubdivide4VectorType<0>,
                 LLVMSubdivide4VectorType<0>],
                [IntrNoMem]>;

  class AdvSIMD_SVE_DOT_Indexed_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMSubdivide4VectorType<0>,
                 LLVMSubdivide4VectorType<0>,
                 llvm_i32_ty],
                [IntrNoMem, ImmArg<ArgIndex<3>>]>;

  class AdvSIMD_SVE_PTEST_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_i1_ty],
                [llvm_anyvector_ty,
                 LLVMMatchType<0>],
                [IntrNoMem]>;

  class AdvSIMD_SVE_TBL_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMVectorOfBitcastsToInt<0>],
                [IntrNoMem]>;

  class AdvSIMD_SVE2_TBX_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMMatchType<0>,
                 LLVMVectorOfBitcastsToInt<0>],
                [IntrNoMem]>;

  class SVE2_1VectorArg_Long_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMSubdivide2VectorType<0>,
                 llvm_i32_ty],
                [IntrNoMem, ImmArg<ArgIndex<1>>]>;

  class SVE2_2VectorArg_Long_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMSubdivide2VectorType<0>,
                 LLVMSubdivide2VectorType<0>],
                [IntrNoMem]>;

  class SVE2_2VectorArgIndexed_Long_Intrinsic
  : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
              [LLVMSubdivide2VectorType<0>,
               LLVMSubdivide2VectorType<0>,
               llvm_i32_ty],
              [IntrNoMem, ImmArg<ArgIndex<2>>]>;

  class SVE2_2VectorArg_Wide_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMSubdivide2VectorType<0>],
                [IntrNoMem]>;

  class SVE2_2VectorArg_Pred_Long_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMMatchType<0>,
                 LLVMSubdivide2VectorType<0>],
                [IntrNoMem]>;

  class SVE2_3VectorArg_Long_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMSubdivide2VectorType<0>,
                 LLVMSubdivide2VectorType<0>],
                [IntrNoMem]>;

  class SVE2_3VectorArgIndexed_Long_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMSubdivide2VectorType<0>,
                 LLVMSubdivide2VectorType<0>,
                 llvm_i32_ty],
                [IntrNoMem, ImmArg<ArgIndex<3>>]>;

  class SVE2_1VectorArg_Narrowing_Intrinsic
    : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
                [llvm_anyvector_ty],
                [IntrNoMem]>;

  class SVE2_Merged1VectorArg_Narrowing_Intrinsic
    : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
                [LLVMSubdivide2VectorType<0>,
                 llvm_anyvector_ty],
                [IntrNoMem]>;
  class SVE2_2VectorArg_Narrowing_Intrinsic
      : DefaultAttrsIntrinsic<
            [LLVMSubdivide2VectorType<0>],
            [llvm_anyvector_ty, LLVMMatchType<0>],
            [IntrNoMem]>;

  class SVE2_Merged2VectorArg_Narrowing_Intrinsic
      : DefaultAttrsIntrinsic<
            [LLVMSubdivide2VectorType<0>],
            [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty, LLVMMatchType<0>],
            [IntrNoMem]>;

  class SVE2_1VectorArg_Imm_Narrowing_Intrinsic
      : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
                  [llvm_anyvector_ty, llvm_i32_ty],
                  [IntrNoMem, ImmArg<ArgIndex<1>>]>;

  class SVE2_2VectorArg_Imm_Narrowing_Intrinsic
      : DefaultAttrsIntrinsic<[LLVMSubdivide2VectorType<0>],
                  [LLVMSubdivide2VectorType<0>, llvm_anyvector_ty,
                   llvm_i32_ty],
                  [IntrNoMem, ImmArg<ArgIndex<2>>]>;

  class SVE2_CONFLICT_DETECT_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMAnyPointerType<llvm_any_ty>,
                 LLVMMatchType<1>]>;

  class SVE2_3VectorArg_Indexed_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMSubdivide2VectorType<0>,
                 LLVMSubdivide2VectorType<0>,
                 llvm_i32_ty],
                [IntrNoMem, ImmArg<ArgIndex<3>>]>;

  class AdvSIMD_SVE_CDOT_LANE_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>,
                 LLVMSubdivide4VectorType<0>,
                 LLVMSubdivide4VectorType<0>,
                 llvm_i32_ty,
                 llvm_i32_ty],
                [IntrNoMem, ImmArg<ArgIndex<3>>, ImmArg<ArgIndex<4>>]>;

  // NOTE: There is no relationship between these intrinsics beyond an attempt
  // to reuse currently identical class definitions.
  class AdvSIMD_SVE_LOGB_Intrinsic  : AdvSIMD_SVE_CNT_Intrinsic;
  class AdvSIMD_SVE2_CADD_Intrinsic : AdvSIMD_2VectorArgIndexed_Intrinsic;
  class AdvSIMD_SVE2_CMLA_Intrinsic : AdvSIMD_3VectorArgIndexed_Intrinsic;

  // This class of intrinsics are not intended to be useful within LLVM IR but
  // are instead here to support some of the more regid parts of the ACLE.
  class Builtin_SVCVT<LLVMType OUT, LLVMType PRED, LLVMType IN>
      : DefaultAttrsIntrinsic<[OUT], [OUT, PRED, IN], [IntrNoMem]>;
}

//===----------------------------------------------------------------------===//
// SVE

let TargetPrefix = "aarch64" in {  // All intrinsics start with "llvm.aarch64.".

class AdvSIMD_SVE_Reduce_Intrinsic
  : DefaultAttrsIntrinsic<[LLVMVectorElementType<0>],
              [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
               llvm_anyvector_ty],
              [IntrNoMem]>;

class AdvSIMD_SVE_SADDV_Reduce_Intrinsic
  : DefaultAttrsIntrinsic<[llvm_i64_ty],
              [LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
               llvm_anyvector_ty],
              [IntrNoMem]>;

class AdvSIMD_SVE_WHILE_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [llvm_anyint_ty, LLVMMatchType<1>],
                [IntrNoMem]>;

class AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [
                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                  LLVMPointerToElt<0>,
                  LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
                ],
                [IntrReadMem, IntrArgMemOnly]>;

class AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [
                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                  LLVMPointerToElt<0>,
                  LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
                ],
                [IntrInaccessibleMemOrArgMemOnly]>;

class AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [
                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                  LLVMPointerToElt<0>,
                  LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
                ],
                [IntrReadMem, IntrArgMemOnly]>;

class AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [
                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                  LLVMPointerToElt<0>,
                  LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
                ],
                [IntrInaccessibleMemOrArgMemOnly]>;

class AdvSIMD_GatherLoad_VS_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [
                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                  llvm_anyvector_ty,
                  llvm_i64_ty
                ],
                [IntrReadMem]>;

class AdvSIMD_GatherLoad_VS_WriteFFR_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [
                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                  llvm_anyvector_ty,
                  llvm_i64_ty
                ],
                [IntrInaccessibleMemOrArgMemOnly]>;

class AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic
    : DefaultAttrsIntrinsic<[],
               [
                 llvm_anyvector_ty,
                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMPointerToElt<0>,
                 LLVMScalarOrSameVectorWidth<0, llvm_i64_ty>
               ],
               [IntrWriteMem, IntrArgMemOnly]>;

class AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic
    : DefaultAttrsIntrinsic<[],
               [
                 llvm_anyvector_ty,
                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 LLVMPointerToElt<0>,
                 LLVMScalarOrSameVectorWidth<0, llvm_i32_ty>
               ],
               [IntrWriteMem, IntrArgMemOnly]>;

class AdvSIMD_ScatterStore_VS_Intrinsic
    : DefaultAttrsIntrinsic<[],
               [
                 llvm_anyvector_ty,
                 LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
                 llvm_anyvector_ty, llvm_i64_ty
               ],
               [IntrWriteMem]>;


class SVE_gather_prf_SV
    : DefaultAttrsIntrinsic<[],
                [
                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate
                  llvm_ptr_ty, // Base address
                  llvm_anyvector_ty, // Offsets
                  llvm_i32_ty // Prfop
                ],
                [IntrInaccessibleMemOrArgMemOnly, NoCapture<ArgIndex<1>>, ImmArg<ArgIndex<3>>]>;

class SVE_gather_prf_VS
    : DefaultAttrsIntrinsic<[],
                [
                  LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, // Predicate
                  llvm_anyvector_ty, // Base addresses
                  llvm_i64_ty, // Scalar offset
                  llvm_i32_ty // Prfop
                ],
                [IntrInaccessibleMemOrArgMemOnly, ImmArg<ArgIndex<3>>]>;

class SVE_MatMul_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                [LLVMMatchType<0>, LLVMSubdivide4VectorType<0>, LLVMSubdivide4VectorType<0>],
                [IntrNoMem]>;

class SVE_4Vec_BF16
    : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty],
                [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty],
                [IntrNoMem]>;

class SVE_4Vec_BF16_Indexed
    : DefaultAttrsIntrinsic<[llvm_nxv4f32_ty],
                [llvm_nxv4f32_ty, llvm_nxv8bf16_ty, llvm_nxv8bf16_ty, llvm_i64_ty],
                [IntrNoMem, ImmArg<ArgIndex<3>>]>;

//
// Vector tuple creation intrinsics (ACLE)
//

def int_aarch64_sve_tuple_create2 : AdvSIMD_SVE_Create_2Vector_Tuple;
def int_aarch64_sve_tuple_create3 : AdvSIMD_SVE_Create_3Vector_Tuple;
def int_aarch64_sve_tuple_create4 : AdvSIMD_SVE_Create_4Vector_Tuple;

//
// Vector tuple insertion/extraction intrinsics (ACLE)
//

def int_aarch64_sve_tuple_get : AdvSIMD_SVE_Get_Vector_Tuple;
def int_aarch64_sve_tuple_set : AdvSIMD_SVE_Set_Vector_Tuple;

//
// Loads
//

def int_aarch64_sve_ld1   : AdvSIMD_1Vec_PredLoad_Intrinsic;

def int_aarch64_sve_ld2 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
def int_aarch64_sve_ld3 : AdvSIMD_ManyVec_PredLoad_Intrinsic;
def int_aarch64_sve_ld4 : AdvSIMD_ManyVec_PredLoad_Intrinsic;

def int_aarch64_sve_ld2_sret : AdvSIMD_2Vec_PredLoad_Intrinsic;
def int_aarch64_sve_ld3_sret : AdvSIMD_3Vec_PredLoad_Intrinsic;
def int_aarch64_sve_ld4_sret : AdvSIMD_4Vec_PredLoad_Intrinsic;

def int_aarch64_sve_ldnt1 : AdvSIMD_1Vec_PredLoad_Intrinsic;
def int_aarch64_sve_ldnf1 : AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic;
def int_aarch64_sve_ldff1 : AdvSIMD_1Vec_PredLoad_WriteFFR_Intrinsic;

def int_aarch64_sve_ld1rq : AdvSIMD_1Vec_PredLoad_Intrinsic;
def int_aarch64_sve_ld1ro : AdvSIMD_1Vec_PredLoad_Intrinsic;

//
// Stores
//

def int_aarch64_sve_st1  : AdvSIMD_1Vec_PredStore_Intrinsic;
def int_aarch64_sve_st2  : AdvSIMD_2Vec_PredStore_Intrinsic;
def int_aarch64_sve_st3  : AdvSIMD_3Vec_PredStore_Intrinsic;
def int_aarch64_sve_st4  : AdvSIMD_4Vec_PredStore_Intrinsic;

def int_aarch64_sve_stnt1 : AdvSIMD_1Vec_PredStore_Intrinsic;

//
// Prefetches
//

def int_aarch64_sve_prf
  : DefaultAttrsIntrinsic<[], [llvm_anyvector_ty, llvm_ptr_ty, llvm_i32_ty],
                  [IntrArgMemOnly, ImmArg<ArgIndex<2>>]>;

// Scalar + 32-bit scaled offset vector, zero extend, packed and
// unpacked.
def int_aarch64_sve_prfb_gather_uxtw_index : SVE_gather_prf_SV;
def int_aarch64_sve_prfh_gather_uxtw_index : SVE_gather_prf_SV;
def int_aarch64_sve_prfw_gather_uxtw_index : SVE_gather_prf_SV;
def int_aarch64_sve_prfd_gather_uxtw_index : SVE_gather_prf_SV;

// Scalar + 32-bit scaled offset vector, sign extend, packed and
// unpacked.
def int_aarch64_sve_prfb_gather_sxtw_index : SVE_gather_prf_SV;
def int_aarch64_sve_prfw_gather_sxtw_index : SVE_gather_prf_SV;
def int_aarch64_sve_prfh_gather_sxtw_index : SVE_gather_prf_SV;
def int_aarch64_sve_prfd_gather_sxtw_index : SVE_gather_prf_SV;

// Scalar + 64-bit scaled offset vector.
def int_aarch64_sve_prfb_gather_index : SVE_gather_prf_SV;
def int_aarch64_sve_prfh_gather_index : SVE_gather_prf_SV;
def int_aarch64_sve_prfw_gather_index : SVE_gather_prf_SV;
def int_aarch64_sve_prfd_gather_index : SVE_gather_prf_SV;

// Vector + scalar.
def int_aarch64_sve_prfb_gather_scalar_offset : SVE_gather_prf_VS;
def int_aarch64_sve_prfh_gather_scalar_offset : SVE_gather_prf_VS;
def int_aarch64_sve_prfw_gather_scalar_offset : SVE_gather_prf_VS;
def int_aarch64_sve_prfd_gather_scalar_offset : SVE_gather_prf_VS;

//
// Scalar to vector operations
//

def int_aarch64_sve_dup : AdvSIMD_SVE_DUP_Intrinsic;
def int_aarch64_sve_dup_x : AdvSIMD_SVE_DUP_Unpred_Intrinsic;

def int_aarch64_sve_index : AdvSIMD_SVE_Index_Intrinsic;

//
// Address calculation
//

def int_aarch64_sve_adrb : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_adrh : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_adrw : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_adrd : AdvSIMD_2VectorArg_Intrinsic;

//
// Integer arithmetic
//

def int_aarch64_sve_add   : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_sub   : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_subr  : AdvSIMD_Pred2VectorArg_Intrinsic;

def int_aarch64_sve_pmul       : AdvSIMD_2VectorArg_Intrinsic;

def int_aarch64_sve_mul        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_mul_lane   : AdvSIMD_2VectorArgIndexed_Intrinsic;
def int_aarch64_sve_smulh      : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_umulh      : AdvSIMD_Pred2VectorArg_Intrinsic;

def int_aarch64_sve_sdiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_udiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_sdivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_udivr      : AdvSIMD_Pred2VectorArg_Intrinsic;

def int_aarch64_sve_smax       : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_umax       : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_smin       : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_umin       : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_sabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_uabd       : AdvSIMD_Pred2VectorArg_Intrinsic;

def int_aarch64_sve_mad        : AdvSIMD_Pred3VectorArg_Intrinsic;
def int_aarch64_sve_msb        : AdvSIMD_Pred3VectorArg_Intrinsic;
def int_aarch64_sve_mla        : AdvSIMD_Pred3VectorArg_Intrinsic;
def int_aarch64_sve_mla_lane   : AdvSIMD_3VectorArgIndexed_Intrinsic;
def int_aarch64_sve_mls        : AdvSIMD_Pred3VectorArg_Intrinsic;
def int_aarch64_sve_mls_lane   : AdvSIMD_3VectorArgIndexed_Intrinsic;

def int_aarch64_sve_saddv      : AdvSIMD_SVE_SADDV_Reduce_Intrinsic;
def int_aarch64_sve_uaddv      : AdvSIMD_SVE_SADDV_Reduce_Intrinsic;

def int_aarch64_sve_smaxv      : AdvSIMD_SVE_Reduce_Intrinsic;
def int_aarch64_sve_umaxv      : AdvSIMD_SVE_Reduce_Intrinsic;
def int_aarch64_sve_sminv      : AdvSIMD_SVE_Reduce_Intrinsic;
def int_aarch64_sve_uminv      : AdvSIMD_SVE_Reduce_Intrinsic;

def int_aarch64_sve_orv        : AdvSIMD_SVE_Reduce_Intrinsic;
def int_aarch64_sve_eorv       : AdvSIMD_SVE_Reduce_Intrinsic;
def int_aarch64_sve_andv       : AdvSIMD_SVE_Reduce_Intrinsic;

def int_aarch64_sve_abs : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_neg : AdvSIMD_Merged1VectorArg_Intrinsic;

def int_aarch64_sve_sdot      : AdvSIMD_SVE_DOT_Intrinsic;
def int_aarch64_sve_sdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;

def int_aarch64_sve_udot      : AdvSIMD_SVE_DOT_Intrinsic;
def int_aarch64_sve_udot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;

def int_aarch64_sve_sqadd_x   : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_sqsub_x   : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_uqadd_x   : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_uqsub_x   : AdvSIMD_2VectorArg_Intrinsic;

// Shifts

def int_aarch64_sve_asr      : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_asr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
def int_aarch64_sve_asrd     : AdvSIMD_SVE_ShiftByImm_Intrinsic;
def int_aarch64_sve_insr     : AdvSIMD_SVE_INSR_Intrinsic;
def int_aarch64_sve_lsl      : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_lsl_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;
def int_aarch64_sve_lsr      : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_lsr_wide : AdvSIMD_SVE_ShiftWide_Intrinsic;

//
// Integer comparisons
//

def int_aarch64_sve_cmpeq : AdvSIMD_SVE_Compare_Intrinsic;
def int_aarch64_sve_cmpge : AdvSIMD_SVE_Compare_Intrinsic;
def int_aarch64_sve_cmpgt : AdvSIMD_SVE_Compare_Intrinsic;
def int_aarch64_sve_cmphi : AdvSIMD_SVE_Compare_Intrinsic;
def int_aarch64_sve_cmphs : AdvSIMD_SVE_Compare_Intrinsic;
def int_aarch64_sve_cmpne : AdvSIMD_SVE_Compare_Intrinsic;

def int_aarch64_sve_cmpeq_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
def int_aarch64_sve_cmpge_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
def int_aarch64_sve_cmpgt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
def int_aarch64_sve_cmphi_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
def int_aarch64_sve_cmphs_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
def int_aarch64_sve_cmple_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
def int_aarch64_sve_cmplo_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
def int_aarch64_sve_cmpls_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
def int_aarch64_sve_cmplt_wide : AdvSIMD_SVE_CompareWide_Intrinsic;
def int_aarch64_sve_cmpne_wide : AdvSIMD_SVE_CompareWide_Intrinsic;

//
// Counting bits
//

def int_aarch64_sve_cls : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_clz : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_cnt : AdvSIMD_SVE_CNT_Intrinsic;

//
// Counting elements
//

def int_aarch64_sve_cntb : AdvSIMD_SVE_CNTB_Intrinsic;
def int_aarch64_sve_cnth : AdvSIMD_SVE_CNTB_Intrinsic;
def int_aarch64_sve_cntw : AdvSIMD_SVE_CNTB_Intrinsic;
def int_aarch64_sve_cntd : AdvSIMD_SVE_CNTB_Intrinsic;

def int_aarch64_sve_cntp : AdvSIMD_SVE_CNTP_Intrinsic;

//
// FFR manipulation
//

def int_aarch64_sve_rdffr   : GCCBuiltin<"__builtin_sve_svrdffr">,   DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [], [IntrReadMem, IntrInaccessibleMemOnly]>;
def int_aarch64_sve_rdffr_z : GCCBuiltin<"__builtin_sve_svrdffr_z">, DefaultAttrsIntrinsic<[llvm_nxv16i1_ty], [llvm_nxv16i1_ty], [IntrReadMem, IntrInaccessibleMemOnly]>;
def int_aarch64_sve_setffr  : GCCBuiltin<"__builtin_sve_svsetffr">,  DefaultAttrsIntrinsic<[], [], [IntrWriteMem, IntrInaccessibleMemOnly]>;
def int_aarch64_sve_wrffr   : GCCBuiltin<"__builtin_sve_svwrffr">,   DefaultAttrsIntrinsic<[], [llvm_nxv16i1_ty], [IntrWriteMem, IntrInaccessibleMemOnly]>;

//
// Saturating scalar arithmetic
//

def int_aarch64_sve_sqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
def int_aarch64_sve_sqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
def int_aarch64_sve_sqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
def int_aarch64_sve_sqdecp : AdvSIMD_SVE_Saturating_Intrinsic;

def int_aarch64_sve_sqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_sqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_sqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_sqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_sqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_sqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_sqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_sqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_sqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_sqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;

def int_aarch64_sve_sqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
def int_aarch64_sve_sqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
def int_aarch64_sve_sqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
def int_aarch64_sve_sqincp : AdvSIMD_SVE_Saturating_Intrinsic;

def int_aarch64_sve_sqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_sqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_sqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_sqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_sqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_sqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_sqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_sqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_sqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_sqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;

def int_aarch64_sve_uqdech : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
def int_aarch64_sve_uqdecw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
def int_aarch64_sve_uqdecd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
def int_aarch64_sve_uqdecp : AdvSIMD_SVE_Saturating_Intrinsic;

def int_aarch64_sve_uqdecb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_uqdecb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_uqdech_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_uqdech_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_uqdecw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_uqdecw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_uqdecd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_uqdecd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_uqdecp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_uqdecp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;

def int_aarch64_sve_uqinch : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
def int_aarch64_sve_uqincw : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
def int_aarch64_sve_uqincd : AdvSIMD_SVE_SaturatingWithPattern_Intrinsic;
def int_aarch64_sve_uqincp : AdvSIMD_SVE_Saturating_Intrinsic;

def int_aarch64_sve_uqincb_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_uqincb_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_uqinch_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_uqinch_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_uqincw_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_uqincw_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_uqincd_n32 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_uqincd_n64 : AdvSIMD_SVE_SaturatingWithPattern_N_Intrinsic<llvm_i64_ty>;
def int_aarch64_sve_uqincp_n32 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i32_ty>;
def int_aarch64_sve_uqincp_n64 : AdvSIMD_SVE_Saturating_N_Intrinsic<llvm_i64_ty>;

//
// Reversal
//

def int_aarch64_sve_rbit : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_revb : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_revh : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_revw : AdvSIMD_Merged1VectorArg_Intrinsic;

//
// Permutations and selection
//

def int_aarch64_sve_clasta    : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_clasta_n  : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
def int_aarch64_sve_clastb    : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_clastb_n  : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
def int_aarch64_sve_compact   : AdvSIMD_Pred1VectorArg_Intrinsic;
def int_aarch64_sve_dupq_lane : AdvSIMD_SVE_DUPQ_Intrinsic;
def int_aarch64_sve_ext       : AdvSIMD_2VectorArgIndexed_Intrinsic;
def int_aarch64_sve_sel       : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_lasta     : AdvSIMD_SVE_Reduce_Intrinsic;
def int_aarch64_sve_lastb     : AdvSIMD_SVE_Reduce_Intrinsic;
def int_aarch64_sve_rev       : AdvSIMD_1VectorArg_Intrinsic;
def int_aarch64_sve_splice    : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_sunpkhi   : AdvSIMD_SVE_Unpack_Intrinsic;
def int_aarch64_sve_sunpklo   : AdvSIMD_SVE_Unpack_Intrinsic;
def int_aarch64_sve_tbl       : AdvSIMD_SVE_TBL_Intrinsic;
def int_aarch64_sve_trn1      : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_trn2      : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_trn1q     : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_trn2q     : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_uunpkhi   : AdvSIMD_SVE_Unpack_Intrinsic;
def int_aarch64_sve_uunpklo   : AdvSIMD_SVE_Unpack_Intrinsic;
def int_aarch64_sve_uzp1      : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_uzp2      : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_uzp1q     : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_uzp2q     : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_zip1      : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_zip2      : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_zip1q     : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_zip2q     : AdvSIMD_2VectorArg_Intrinsic;

//
// Logical operations
//

def int_aarch64_sve_and  : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_bic  : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_cnot : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_eor  : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_not  : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_orr  : AdvSIMD_Pred2VectorArg_Intrinsic;

//
// Conversion
//

def int_aarch64_sve_sxtb : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_sxth : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_sxtw : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_uxtb : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_uxth : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_uxtw : AdvSIMD_Merged1VectorArg_Intrinsic;

//
// While comparisons
//

def int_aarch64_sve_whilele : AdvSIMD_SVE_WHILE_Intrinsic;
def int_aarch64_sve_whilelo : AdvSIMD_SVE_WHILE_Intrinsic;
def int_aarch64_sve_whilels : AdvSIMD_SVE_WHILE_Intrinsic;
def int_aarch64_sve_whilelt : AdvSIMD_SVE_WHILE_Intrinsic;
def int_aarch64_sve_whilege : AdvSIMD_SVE_WHILE_Intrinsic;
def int_aarch64_sve_whilegt : AdvSIMD_SVE_WHILE_Intrinsic;
def int_aarch64_sve_whilehs : AdvSIMD_SVE_WHILE_Intrinsic;
def int_aarch64_sve_whilehi : AdvSIMD_SVE_WHILE_Intrinsic;

//
// Floating-point arithmetic
//

def int_aarch64_sve_fabd       : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_fabs       : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_fadd       : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_fcadd      : AdvSIMD_SVE_CADD_Intrinsic;
def int_aarch64_sve_fcmla      : AdvSIMD_SVE_CMLA_Intrinsic;
def int_aarch64_sve_fcmla_lane : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
def int_aarch64_sve_fdiv       : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_fdivr      : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_fexpa_x    : AdvSIMD_SVE_EXPA_Intrinsic;
def int_aarch64_sve_fmad       : AdvSIMD_Pred3VectorArg_Intrinsic;
def int_aarch64_sve_fmax       : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_fmaxnm     : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_fmin       : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_fminnm     : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_fmla       : AdvSIMD_Pred3VectorArg_Intrinsic;
def int_aarch64_sve_fmla_lane  : AdvSIMD_3VectorArgIndexed_Intrinsic;
def int_aarch64_sve_fmls       : AdvSIMD_Pred3VectorArg_Intrinsic;
def int_aarch64_sve_fmls_lane  : AdvSIMD_3VectorArgIndexed_Intrinsic;
def int_aarch64_sve_fmsb       : AdvSIMD_Pred3VectorArg_Intrinsic;
def int_aarch64_sve_fmul       : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_fmulx      : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_fneg       : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_fmul_lane  : AdvSIMD_2VectorArgIndexed_Intrinsic;
def int_aarch64_sve_fnmad      : AdvSIMD_Pred3VectorArg_Intrinsic;
def int_aarch64_sve_fnmla      : AdvSIMD_Pred3VectorArg_Intrinsic;
def int_aarch64_sve_fnmls      : AdvSIMD_Pred3VectorArg_Intrinsic;
def int_aarch64_sve_fnmsb      : AdvSIMD_Pred3VectorArg_Intrinsic;
def int_aarch64_sve_frecpe_x   : AdvSIMD_1VectorArg_Intrinsic;
def int_aarch64_sve_frecps_x   : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_frecpx     : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_frinta     : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_frinti     : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_frintm     : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_frintn     : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_frintp     : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_frintx     : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_frintz     : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_frsqrte_x  : AdvSIMD_1VectorArg_Intrinsic;
def int_aarch64_sve_frsqrts_x  : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_fscale     : AdvSIMD_SVE_SCALE_Intrinsic;
def int_aarch64_sve_fsqrt      : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_fsub       : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_fsubr      : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_ftmad_x    : AdvSIMD_2VectorArgIndexed_Intrinsic;
def int_aarch64_sve_ftsmul_x   : AdvSIMD_SVE_TSMUL_Intrinsic;
def int_aarch64_sve_ftssel_x   : AdvSIMD_SVE_TSMUL_Intrinsic;

//
// Floating-point reductions
//

def int_aarch64_sve_fadda   : AdvSIMD_SVE_ReduceWithInit_Intrinsic;
def int_aarch64_sve_faddv   : AdvSIMD_SVE_Reduce_Intrinsic;
def int_aarch64_sve_fmaxv   : AdvSIMD_SVE_Reduce_Intrinsic;
def int_aarch64_sve_fmaxnmv : AdvSIMD_SVE_Reduce_Intrinsic;
def int_aarch64_sve_fminv   : AdvSIMD_SVE_Reduce_Intrinsic;
def int_aarch64_sve_fminnmv : AdvSIMD_SVE_Reduce_Intrinsic;

//
// Floating-point conversions
//

def int_aarch64_sve_fcvt   : AdvSIMD_SVE_FCVT_Intrinsic;
def int_aarch64_sve_fcvtzs : AdvSIMD_SVE_FCVTZS_Intrinsic;
def int_aarch64_sve_fcvtzu : AdvSIMD_SVE_FCVTZS_Intrinsic;
def int_aarch64_sve_scvtf  : AdvSIMD_SVE_SCVTF_Intrinsic;
def int_aarch64_sve_ucvtf  : AdvSIMD_SVE_SCVTF_Intrinsic;

//
// Floating-point comparisons
//

def int_aarch64_sve_facge : AdvSIMD_SVE_Compare_Intrinsic;
def int_aarch64_sve_facgt : AdvSIMD_SVE_Compare_Intrinsic;

def int_aarch64_sve_fcmpeq : AdvSIMD_SVE_Compare_Intrinsic;
def int_aarch64_sve_fcmpge : AdvSIMD_SVE_Compare_Intrinsic;
def int_aarch64_sve_fcmpgt : AdvSIMD_SVE_Compare_Intrinsic;
def int_aarch64_sve_fcmpne : AdvSIMD_SVE_Compare_Intrinsic;
def int_aarch64_sve_fcmpuo : AdvSIMD_SVE_Compare_Intrinsic;

def int_aarch64_sve_fcvtzs_i32f16   : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
def int_aarch64_sve_fcvtzs_i32f64   : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
def int_aarch64_sve_fcvtzs_i64f16   : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
def int_aarch64_sve_fcvtzs_i64f32   : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;

def int_aarch64_sve_fcvt_bf16f32    : Builtin_SVCVT<llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;
def int_aarch64_sve_fcvtnt_bf16f32  : Builtin_SVCVT<llvm_nxv8bf16_ty, llvm_nxv8i1_ty, llvm_nxv4f32_ty>;

def int_aarch64_sve_fcvtzu_i32f16   : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
def int_aarch64_sve_fcvtzu_i32f64   : Builtin_SVCVT<llvm_nxv4i32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
def int_aarch64_sve_fcvtzu_i64f16   : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
def int_aarch64_sve_fcvtzu_i64f32   : Builtin_SVCVT<llvm_nxv2i64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;

def int_aarch64_sve_fcvt_f16f32     : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
def int_aarch64_sve_fcvt_f16f64     : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
def int_aarch64_sve_fcvt_f32f64     : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;

def int_aarch64_sve_fcvt_f32f16     : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
def int_aarch64_sve_fcvt_f64f16     : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv8f16_ty>;
def int_aarch64_sve_fcvt_f64f32     : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;

def int_aarch64_sve_fcvtlt_f32f16   : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv4i1_ty, llvm_nxv8f16_ty>;
def int_aarch64_sve_fcvtlt_f64f32   : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4f32_ty>;
def int_aarch64_sve_fcvtnt_f16f32   : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4f32_ty>;
def int_aarch64_sve_fcvtnt_f32f64   : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;

def int_aarch64_sve_fcvtx_f32f64    : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;
def int_aarch64_sve_fcvtxnt_f32f64  : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2f64_ty>;

def int_aarch64_sve_scvtf_f16i32    : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>;
def int_aarch64_sve_scvtf_f16i64    : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
def int_aarch64_sve_scvtf_f32i64    : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
def int_aarch64_sve_scvtf_f64i32    : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>;

def int_aarch64_sve_ucvtf_f16i32    : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv4i1_ty, llvm_nxv4i32_ty>;
def int_aarch64_sve_ucvtf_f16i64    : Builtin_SVCVT<llvm_nxv8f16_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
def int_aarch64_sve_ucvtf_f32i64    : Builtin_SVCVT<llvm_nxv4f32_ty, llvm_nxv2i1_ty, llvm_nxv2i64_ty>;
def int_aarch64_sve_ucvtf_f64i32    : Builtin_SVCVT<llvm_nxv2f64_ty, llvm_nxv2i1_ty, llvm_nxv4i32_ty>;

//
// Predicate creation
//

def int_aarch64_sve_ptrue : AdvSIMD_SVE_PTRUE_Intrinsic;

//
// Predicate operations
//

def int_aarch64_sve_and_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_bic_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_brka    : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_brka_z  : AdvSIMD_Pred1VectorArg_Intrinsic;
def int_aarch64_sve_brkb    : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_brkb_z  : AdvSIMD_Pred1VectorArg_Intrinsic;
def int_aarch64_sve_brkn_z  : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_brkpa_z : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_brkpb_z : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_eor_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_nand_z  : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_nor_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_orn_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_orr_z   : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_pfirst  : AdvSIMD_Pred1VectorArg_Intrinsic;
def int_aarch64_sve_pnext   : AdvSIMD_Pred1VectorArg_Intrinsic;
def int_aarch64_sve_punpkhi : AdvSIMD_SVE_PUNPKHI_Intrinsic;
def int_aarch64_sve_punpklo : AdvSIMD_SVE_PUNPKHI_Intrinsic;

//
// Testing predicates
//

def int_aarch64_sve_ptest_any   : AdvSIMD_SVE_PTEST_Intrinsic;
def int_aarch64_sve_ptest_first : AdvSIMD_SVE_PTEST_Intrinsic;
def int_aarch64_sve_ptest_last  : AdvSIMD_SVE_PTEST_Intrinsic;

//
// Reinterpreting data
//

def int_aarch64_sve_convert_from_svbool : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
                                                    [llvm_nxv16i1_ty],
                                                    [IntrNoMem]>;

def int_aarch64_sve_convert_to_svbool : DefaultAttrsIntrinsic<[llvm_nxv16i1_ty],
                                                  [llvm_anyvector_ty],
                                                  [IntrNoMem]>;

//
// Gather loads: scalar base + vector offsets
//

// 64 bit unscaled offsets
def int_aarch64_sve_ld1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;

// 64 bit scaled offsets
def int_aarch64_sve_ld1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;

// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
def int_aarch64_sve_ld1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
def int_aarch64_sve_ld1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;

// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
def int_aarch64_sve_ld1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;
def int_aarch64_sve_ld1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;

//
// Gather loads: vector base + scalar offset
//

def int_aarch64_sve_ld1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_Intrinsic;


//
// First-faulting gather loads: scalar base + vector offsets
//

// 64 bit unscaled offsets
def int_aarch64_sve_ldff1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic;

// 64 bit scaled offsets
def int_aarch64_sve_ldff1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_WriteFFR_Intrinsic;

// 32 bit unscaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits
def int_aarch64_sve_ldff1_gather_sxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic;
def int_aarch64_sve_ldff1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic;

// 32 bit scaled offsets, sign (sxtw) or zero (uxtw) extended to 64 bits
def int_aarch64_sve_ldff1_gather_sxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic;
def int_aarch64_sve_ldff1_gather_uxtw_index : AdvSIMD_GatherLoad_SV_32b_Offsets_WriteFFR_Intrinsic;

//
// First-faulting gather loads: vector base + scalar offset
//

def int_aarch64_sve_ldff1_gather_scalar_offset : AdvSIMD_GatherLoad_VS_WriteFFR_Intrinsic;


//
// Non-temporal gather loads: scalar base + vector offsets
//

// 64 bit unscaled offsets
def int_aarch64_sve_ldnt1_gather : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;

// 64 bit indices
def int_aarch64_sve_ldnt1_gather_index : AdvSIMD_GatherLoad_SV_64b_Offsets_Intrinsic;

// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits
def int_aarch64_sve_ldnt1_gather_uxtw : AdvSIMD_GatherLoad_SV_32b_Offsets_Intrinsic;

//
// Non-temporal gather loads: vector base + scalar offset
//

def int_aarch64_sve_ldnt1_gather_scalar_offset  : AdvSIMD_GatherLoad_VS_Intrinsic;

//
// Scatter stores: scalar base + vector offsets
//

// 64 bit unscaled offsets
def int_aarch64_sve_st1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;

// 64 bit scaled offsets
def int_aarch64_sve_st1_scatter_index
    : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;

// 32 bit unscaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
def int_aarch64_sve_st1_scatter_sxtw
    : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;

def int_aarch64_sve_st1_scatter_uxtw
    : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;

// 32 bit scaled offsets, sign (sxtw) or zero (zxtw) extended to 64 bits
def int_aarch64_sve_st1_scatter_sxtw_index
    : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;

def int_aarch64_sve_st1_scatter_uxtw_index
    : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;

//
// Scatter stores: vector base + scalar offset
//

def int_aarch64_sve_st1_scatter_scalar_offset : AdvSIMD_ScatterStore_VS_Intrinsic;

//
// Non-temporal scatter stores: scalar base + vector offsets
//

// 64 bit unscaled offsets
def int_aarch64_sve_stnt1_scatter : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;

// 64 bit indices
def int_aarch64_sve_stnt1_scatter_index
    : AdvSIMD_ScatterStore_SV_64b_Offsets_Intrinsic;

// 32 bit unscaled offsets, zero (zxtw) extended to 64 bits
def int_aarch64_sve_stnt1_scatter_uxtw : AdvSIMD_ScatterStore_SV_32b_Offsets_Intrinsic;

//
// Non-temporal scatter stores: vector base + scalar offset
//

def int_aarch64_sve_stnt1_scatter_scalar_offset  : AdvSIMD_ScatterStore_VS_Intrinsic;

//
// SVE2 - Uniform DSP operations
//

def int_aarch64_sve_saba          : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_shadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_shsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_shsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_sli           : AdvSIMD_2VectorArgIndexed_Intrinsic;
def int_aarch64_sve_sqabs         : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_sqadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_sqdmulh       : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_sqdmulh_lane  : AdvSIMD_2VectorArgIndexed_Intrinsic;
def int_aarch64_sve_sqneg         : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_sqrdmlah      : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_sqrdmlah_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
def int_aarch64_sve_sqrdmlsh      : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_sqrdmlsh_lane : AdvSIMD_3VectorArgIndexed_Intrinsic;
def int_aarch64_sve_sqrdmulh      : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_sqrdmulh_lane : AdvSIMD_2VectorArgIndexed_Intrinsic;
def int_aarch64_sve_sqrshl        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_sqshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_sqshlu        : AdvSIMD_SVE_ShiftByImm_Intrinsic;
def int_aarch64_sve_sqsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_sqsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_srhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_sri           : AdvSIMD_2VectorArgIndexed_Intrinsic;
def int_aarch64_sve_srshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_srshr         : AdvSIMD_SVE_ShiftByImm_Intrinsic;
def int_aarch64_sve_srsra         : AdvSIMD_2VectorArgIndexed_Intrinsic;
def int_aarch64_sve_ssra          : AdvSIMD_2VectorArgIndexed_Intrinsic;
def int_aarch64_sve_suqadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_uaba          : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_uhadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_uhsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_uhsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_uqadd         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_uqrshl        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_uqshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_uqsub         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_uqsubr        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_urecpe        : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_urhadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_urshl         : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_urshr         : AdvSIMD_SVE_ShiftByImm_Intrinsic;
def int_aarch64_sve_ursqrte       : AdvSIMD_Merged1VectorArg_Intrinsic;
def int_aarch64_sve_ursra         : AdvSIMD_2VectorArgIndexed_Intrinsic;
def int_aarch64_sve_usqadd        : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_usra          : AdvSIMD_2VectorArgIndexed_Intrinsic;

//
// SVE2 - Widening DSP operations
//

def int_aarch64_sve_sabalb : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_sabalt : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_sabdlb : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_sabdlt : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_saddlb : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_saddlt : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_saddwb : SVE2_2VectorArg_Wide_Intrinsic;
def int_aarch64_sve_saddwt : SVE2_2VectorArg_Wide_Intrinsic;
def int_aarch64_sve_sshllb : SVE2_1VectorArg_Long_Intrinsic;
def int_aarch64_sve_sshllt : SVE2_1VectorArg_Long_Intrinsic;
def int_aarch64_sve_ssublb : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_ssublt : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_ssubwb : SVE2_2VectorArg_Wide_Intrinsic;
def int_aarch64_sve_ssubwt : SVE2_2VectorArg_Wide_Intrinsic;
def int_aarch64_sve_uabalb : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_uabalt : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_uabdlb : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_uabdlt : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_uaddlb : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_uaddlt : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_uaddwb : SVE2_2VectorArg_Wide_Intrinsic;
def int_aarch64_sve_uaddwt : SVE2_2VectorArg_Wide_Intrinsic;
def int_aarch64_sve_ushllb : SVE2_1VectorArg_Long_Intrinsic;
def int_aarch64_sve_ushllt : SVE2_1VectorArg_Long_Intrinsic;
def int_aarch64_sve_usublb : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_usublt : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_usubwb : SVE2_2VectorArg_Wide_Intrinsic;
def int_aarch64_sve_usubwt : SVE2_2VectorArg_Wide_Intrinsic;

//
// SVE2 - Non-widening pairwise arithmetic
//

def int_aarch64_sve_addp    : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_faddp   : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_fmaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_fmaxnmp : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_fminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_fminnmp : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_smaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_sminp   : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_umaxp   : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_uminp   : AdvSIMD_Pred2VectorArg_Intrinsic;

//
// SVE2 - Widening pairwise arithmetic
//

def int_aarch64_sve_sadalp : SVE2_2VectorArg_Pred_Long_Intrinsic;
def int_aarch64_sve_uadalp : SVE2_2VectorArg_Pred_Long_Intrinsic;

//
// SVE2 - Uniform complex integer arithmetic
//

def int_aarch64_sve_cadd_x           : AdvSIMD_SVE2_CADD_Intrinsic;
def int_aarch64_sve_sqcadd_x         : AdvSIMD_SVE2_CADD_Intrinsic;
def int_aarch64_sve_cmla_x           : AdvSIMD_SVE2_CMLA_Intrinsic;
def int_aarch64_sve_cmla_lane_x      : AdvSIMD_SVE_CMLA_LANE_Intrinsic;
def int_aarch64_sve_sqrdcmlah_x      : AdvSIMD_SVE2_CMLA_Intrinsic;
def int_aarch64_sve_sqrdcmlah_lane_x : AdvSIMD_SVE_CMLA_LANE_Intrinsic;

//
// SVE2 - Widening complex integer arithmetic
//

def int_aarch64_sve_saddlbt   : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_ssublbt   : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_ssubltb   : SVE2_2VectorArg_Long_Intrinsic;

//
// SVE2 - Widening complex integer dot product
//

def int_aarch64_sve_cdot      : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
def int_aarch64_sve_cdot_lane : AdvSIMD_SVE_CDOT_LANE_Intrinsic;

//
// SVE2 - Floating-point widening multiply-accumulate
//

def int_aarch64_sve_fmlalb        : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_fmlalb_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
def int_aarch64_sve_fmlalt        : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_fmlalt_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
def int_aarch64_sve_fmlslb        : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_fmlslb_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;
def int_aarch64_sve_fmlslt        : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_fmlslt_lane   : SVE2_3VectorArgIndexed_Long_Intrinsic;

//
// SVE2 - Floating-point integer binary logarithm
//

def int_aarch64_sve_flogb : AdvSIMD_SVE_LOGB_Intrinsic;

//
// SVE2 - Vector histogram count
//

def int_aarch64_sve_histcnt : AdvSIMD_Pred2VectorArg_Intrinsic;
def int_aarch64_sve_histseg : AdvSIMD_2VectorArg_Intrinsic;

//
// SVE2 - Character match
//

def int_aarch64_sve_match   : AdvSIMD_SVE_Compare_Intrinsic;
def int_aarch64_sve_nmatch  : AdvSIMD_SVE_Compare_Intrinsic;

//
// SVE2 - Unary narrowing operations
//

def int_aarch64_sve_sqxtnb  : SVE2_1VectorArg_Narrowing_Intrinsic;
def int_aarch64_sve_sqxtnt  : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
def int_aarch64_sve_sqxtunb : SVE2_1VectorArg_Narrowing_Intrinsic;
def int_aarch64_sve_sqxtunt : SVE2_Merged1VectorArg_Narrowing_Intrinsic;
def int_aarch64_sve_uqxtnb  : SVE2_1VectorArg_Narrowing_Intrinsic;
def int_aarch64_sve_uqxtnt  : SVE2_Merged1VectorArg_Narrowing_Intrinsic;

//
// SVE2 - Binary narrowing DSP operations
//
def int_aarch64_sve_addhnb    : SVE2_2VectorArg_Narrowing_Intrinsic;
def int_aarch64_sve_addhnt    : SVE2_Merged2VectorArg_Narrowing_Intrinsic;

def int_aarch64_sve_raddhnb   : SVE2_2VectorArg_Narrowing_Intrinsic;
def int_aarch64_sve_raddhnt   : SVE2_Merged2VectorArg_Narrowing_Intrinsic;

def int_aarch64_sve_subhnb    : SVE2_2VectorArg_Narrowing_Intrinsic;
def int_aarch64_sve_subhnt    : SVE2_Merged2VectorArg_Narrowing_Intrinsic;

def int_aarch64_sve_rsubhnb   : SVE2_2VectorArg_Narrowing_Intrinsic;
def int_aarch64_sve_rsubhnt   : SVE2_Merged2VectorArg_Narrowing_Intrinsic;

// Narrowing shift right
def int_aarch64_sve_shrnb     : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
def int_aarch64_sve_shrnt     : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;

def int_aarch64_sve_rshrnb    : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
def int_aarch64_sve_rshrnt    : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;

// Saturating shift right - signed input/output
def int_aarch64_sve_sqshrnb   : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
def int_aarch64_sve_sqshrnt   : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;

def int_aarch64_sve_sqrshrnb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
def int_aarch64_sve_sqrshrnt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;

// Saturating shift right - unsigned input/output
def int_aarch64_sve_uqshrnb   : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
def int_aarch64_sve_uqshrnt   : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;

def int_aarch64_sve_uqrshrnb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
def int_aarch64_sve_uqrshrnt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;

// Saturating shift right - signed input, unsigned output
def int_aarch64_sve_sqshrunb  : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
def int_aarch64_sve_sqshrunt  : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;

def int_aarch64_sve_sqrshrunb : SVE2_1VectorArg_Imm_Narrowing_Intrinsic;
def int_aarch64_sve_sqrshrunt : SVE2_2VectorArg_Imm_Narrowing_Intrinsic;

// SVE2 MLA LANE.
def int_aarch64_sve_smlalb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
def int_aarch64_sve_smlalt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
def int_aarch64_sve_umlalb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
def int_aarch64_sve_umlalt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
def int_aarch64_sve_smlslb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
def int_aarch64_sve_smlslt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
def int_aarch64_sve_umlslb_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
def int_aarch64_sve_umlslt_lane   : SVE2_3VectorArg_Indexed_Intrinsic;
def int_aarch64_sve_smullb_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
def int_aarch64_sve_smullt_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
def int_aarch64_sve_umullb_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
def int_aarch64_sve_umullt_lane   : SVE2_2VectorArgIndexed_Long_Intrinsic;
def int_aarch64_sve_sqdmlalb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
def int_aarch64_sve_sqdmlalt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
def int_aarch64_sve_sqdmlslb_lane : SVE2_3VectorArg_Indexed_Intrinsic;
def int_aarch64_sve_sqdmlslt_lane : SVE2_3VectorArg_Indexed_Intrinsic;
def int_aarch64_sve_sqdmullb_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;
def int_aarch64_sve_sqdmullt_lane : SVE2_2VectorArgIndexed_Long_Intrinsic;

// SVE2 MLA Unpredicated.
def int_aarch64_sve_smlalb      : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_smlalt      : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_umlalb      : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_umlalt      : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_smlslb      : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_smlslt      : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_umlslb      : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_umlslt      : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_smullb      : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_smullt      : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_umullb      : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_umullt      : SVE2_2VectorArg_Long_Intrinsic;

def int_aarch64_sve_sqdmlalb    : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_sqdmlalt    : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_sqdmlslb    : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_sqdmlslt    : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_sqdmullb    : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_sqdmullt    : SVE2_2VectorArg_Long_Intrinsic;
def int_aarch64_sve_sqdmlalbt   : SVE2_3VectorArg_Long_Intrinsic;
def int_aarch64_sve_sqdmlslbt   : SVE2_3VectorArg_Long_Intrinsic;

// SVE2 ADDSUB Long Unpredicated.
def int_aarch64_sve_adclb       : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_adclt       : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_sbclb       : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_sbclt       : AdvSIMD_3VectorArg_Intrinsic;

//
// SVE2 - Polynomial arithmetic
//
def int_aarch64_sve_eorbt       : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_eortb       : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_pmullb_pair : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_pmullt_pair : AdvSIMD_2VectorArg_Intrinsic;

//
// SVE2 bitwise ternary operations.
//
def int_aarch64_sve_eor3   : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_bcax   : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_bsl    : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_bsl1n  : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_bsl2n  : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_nbsl   : AdvSIMD_3VectorArg_Intrinsic;
def int_aarch64_sve_xar    : AdvSIMD_2VectorArgIndexed_Intrinsic;

//
// SVE2 - Optional AES, SHA-3 and SM4
//

def int_aarch64_sve_aesd    : GCCBuiltin<"__builtin_sve_svaesd_u8">,
                              DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
                                        [llvm_nxv16i8_ty, llvm_nxv16i8_ty],
                                        [IntrNoMem]>;
def int_aarch64_sve_aesimc  : GCCBuiltin<"__builtin_sve_svaesimc_u8">,
                              DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
                                        [llvm_nxv16i8_ty],
                                        [IntrNoMem]>;
def int_aarch64_sve_aese    : GCCBuiltin<"__builtin_sve_svaese_u8">,
                              DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
                                        [llvm_nxv16i8_ty, llvm_nxv16i8_ty],
                                        [IntrNoMem]>;
def int_aarch64_sve_aesmc   : GCCBuiltin<"__builtin_sve_svaesmc_u8">,
                              DefaultAttrsIntrinsic<[llvm_nxv16i8_ty],
                                        [llvm_nxv16i8_ty],
                                        [IntrNoMem]>;
def int_aarch64_sve_rax1    : GCCBuiltin<"__builtin_sve_svrax1_u64">,
                              DefaultAttrsIntrinsic<[llvm_nxv2i64_ty],
                                        [llvm_nxv2i64_ty, llvm_nxv2i64_ty],
                                        [IntrNoMem]>;
def int_aarch64_sve_sm4e    : GCCBuiltin<"__builtin_sve_svsm4e_u32">,
                              DefaultAttrsIntrinsic<[llvm_nxv4i32_ty],
                                        [llvm_nxv4i32_ty, llvm_nxv4i32_ty],
                                        [IntrNoMem]>;
def int_aarch64_sve_sm4ekey : GCCBuiltin<"__builtin_sve_svsm4ekey_u32">,
                              DefaultAttrsIntrinsic<[llvm_nxv4i32_ty],
                                        [llvm_nxv4i32_ty, llvm_nxv4i32_ty],
                                        [IntrNoMem]>;
//
// SVE2 - Extended table lookup/permute
//

def int_aarch64_sve_tbl2 : AdvSIMD_SVE2_TBX_Intrinsic;
def int_aarch64_sve_tbx  : AdvSIMD_SVE2_TBX_Intrinsic;

//
// SVE2 - Optional bit permutation
//

def int_aarch64_sve_bdep_x : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_bext_x : AdvSIMD_2VectorArg_Intrinsic;
def int_aarch64_sve_bgrp_x : AdvSIMD_2VectorArg_Intrinsic;


//
// SVE ACLE: 7.3. INT8 matrix multiply extensions
//
def int_aarch64_sve_ummla : SVE_MatMul_Intrinsic;
def int_aarch64_sve_smmla : SVE_MatMul_Intrinsic;
def int_aarch64_sve_usmmla : SVE_MatMul_Intrinsic;

def int_aarch64_sve_usdot : AdvSIMD_SVE_DOT_Intrinsic;
def int_aarch64_sve_usdot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;
def int_aarch64_sve_sudot_lane : AdvSIMD_SVE_DOT_Indexed_Intrinsic;

//
// SVE ACLE: 7.4/5. FP64/FP32 matrix multiply extensions
//
def int_aarch64_sve_fmmla : AdvSIMD_3VectorArg_Intrinsic;

//
// SVE ACLE: 7.2. BFloat16 extensions
//

def int_aarch64_sve_bfdot   : SVE_4Vec_BF16;
def int_aarch64_sve_bfmlalb : SVE_4Vec_BF16;
def int_aarch64_sve_bfmlalt : SVE_4Vec_BF16;

def int_aarch64_sve_bfmmla  : SVE_4Vec_BF16;

def int_aarch64_sve_bfdot_lane   : SVE_4Vec_BF16_Indexed;
def int_aarch64_sve_bfmlalb_lane : SVE_4Vec_BF16_Indexed;
def int_aarch64_sve_bfmlalt_lane : SVE_4Vec_BF16_Indexed;
}

//
// SVE2 - Contiguous conflict detection
//

def int_aarch64_sve_whilerw_b : SVE2_CONFLICT_DETECT_Intrinsic;
def int_aarch64_sve_whilerw_h : SVE2_CONFLICT_DETECT_Intrinsic;
def int_aarch64_sve_whilerw_s : SVE2_CONFLICT_DETECT_Intrinsic;
def int_aarch64_sve_whilerw_d : SVE2_CONFLICT_DETECT_Intrinsic;
def int_aarch64_sve_whilewr_b : SVE2_CONFLICT_DETECT_Intrinsic;
def int_aarch64_sve_whilewr_h : SVE2_CONFLICT_DETECT_Intrinsic;
def int_aarch64_sve_whilewr_s : SVE2_CONFLICT_DETECT_Intrinsic;
def int_aarch64_sve_whilewr_d : SVE2_CONFLICT_DETECT_Intrinsic;

// Scalable Matrix Extension (SME) Intrinsics
let TargetPrefix = "aarch64" in {
  class SME_Load_Store_B_Intrinsic
    : DefaultAttrsIntrinsic<[],
        [llvm_nxv16i1_ty, llvm_ptr_ty, llvm_i64_ty, llvm_i32_ty], []>;
  class SME_Load_Store_H_Intrinsic
    : DefaultAttrsIntrinsic<[],
        [llvm_nxv16i1_ty, LLVMPointerType<llvm_i16_ty>, llvm_i64_ty, llvm_i32_ty], []>;
  class SME_Load_Store_S_Intrinsic
    : DefaultAttrsIntrinsic<[],
        [llvm_nxv16i1_ty, LLVMPointerType<llvm_i32_ty>, llvm_i64_ty, llvm_i32_ty], []>;
  class SME_Load_Store_D_Intrinsic
    : DefaultAttrsIntrinsic<[],
        [llvm_nxv16i1_ty, LLVMPointerType<llvm_i64_ty>, llvm_i64_ty, llvm_i32_ty], []>;
  class SME_Load_Store_Q_Intrinsic
    : DefaultAttrsIntrinsic<[],
        [llvm_nxv16i1_ty, LLVMPointerType<llvm_i128_ty>, llvm_i64_ty, llvm_i32_ty], []>;

  // Loads
  def int_aarch64_sme_ld1b_horiz : SME_Load_Store_B_Intrinsic;
  def int_aarch64_sme_ld1h_horiz : SME_Load_Store_H_Intrinsic;
  def int_aarch64_sme_ld1w_horiz : SME_Load_Store_S_Intrinsic;
  def int_aarch64_sme_ld1d_horiz : SME_Load_Store_D_Intrinsic;
  def int_aarch64_sme_ld1q_horiz : SME_Load_Store_Q_Intrinsic;
  def int_aarch64_sme_ld1b_vert  : SME_Load_Store_B_Intrinsic;
  def int_aarch64_sme_ld1h_vert  : SME_Load_Store_H_Intrinsic;
  def int_aarch64_sme_ld1w_vert  : SME_Load_Store_S_Intrinsic;
  def int_aarch64_sme_ld1d_vert  : SME_Load_Store_D_Intrinsic;
  def int_aarch64_sme_ld1q_vert  : SME_Load_Store_Q_Intrinsic;

  // Stores
  def int_aarch64_sme_st1b_horiz : SME_Load_Store_B_Intrinsic;
  def int_aarch64_sme_st1h_horiz : SME_Load_Store_H_Intrinsic;
  def int_aarch64_sme_st1w_horiz : SME_Load_Store_S_Intrinsic;
  def int_aarch64_sme_st1d_horiz : SME_Load_Store_D_Intrinsic;
  def int_aarch64_sme_st1q_horiz : SME_Load_Store_Q_Intrinsic;
  def int_aarch64_sme_st1b_vert  : SME_Load_Store_B_Intrinsic;
  def int_aarch64_sme_st1h_vert  : SME_Load_Store_H_Intrinsic;
  def int_aarch64_sme_st1w_vert  : SME_Load_Store_S_Intrinsic;
  def int_aarch64_sme_st1d_vert  : SME_Load_Store_D_Intrinsic;
  def int_aarch64_sme_st1q_vert  : SME_Load_Store_Q_Intrinsic;

  // Spill + fill
  def int_aarch64_sme_ldr : DefaultAttrsIntrinsic<
    [], [llvm_i32_ty, llvm_ptr_ty]>;
  def int_aarch64_sme_str : DefaultAttrsIntrinsic<
    [], [llvm_i32_ty, llvm_ptr_ty]>;

  class SME_TileToVector_Intrinsic
      : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
          [LLVMMatchType<0>, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>, llvm_i64_ty, llvm_i32_ty]>;
  class SME_VectorToTile_Intrinsic
      : DefaultAttrsIntrinsic<[],
          [llvm_i64_ty, llvm_i32_ty, LLVMScalarOrSameVectorWidth<0, llvm_i1_ty>,
           llvm_anyvector_ty]>;

  def int_aarch64_sme_read_horiz  : SME_TileToVector_Intrinsic;
  def int_aarch64_sme_read_vert   : SME_TileToVector_Intrinsic;
  def int_aarch64_sme_write_horiz : SME_VectorToTile_Intrinsic;
  def int_aarch64_sme_write_vert  : SME_VectorToTile_Intrinsic;

  def int_aarch64_sme_readq_horiz  : SME_TileToVector_Intrinsic;
  def int_aarch64_sme_readq_vert   : SME_TileToVector_Intrinsic;
  def int_aarch64_sme_writeq_horiz : SME_VectorToTile_Intrinsic;
  def int_aarch64_sme_writeq_vert  : SME_VectorToTile_Intrinsic;


  //
  // Counting elements
  //

  class AdvSIMD_SME_CNTSB_Intrinsic
    : DefaultAttrsIntrinsic<[llvm_i64_ty], [], [IntrNoMem]>;

  def int_aarch64_sme_cntsb : AdvSIMD_SME_CNTSB_Intrinsic;
  def int_aarch64_sme_cntsh : AdvSIMD_SME_CNTSB_Intrinsic;
  def int_aarch64_sme_cntsw : AdvSIMD_SME_CNTSB_Intrinsic;
  def int_aarch64_sme_cntsd : AdvSIMD_SME_CNTSB_Intrinsic;
}