//
// Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, 2022, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License version 2 only, as
// published by the Free Software Foundation.
//
// This code is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
// version 2 for more details (a copy is included in the LICENSE file that
// accompanied this code).
//
// You should have received a copy of the GNU General Public License version
// 2 along with this work; if not, write to the Free Software Foundation,
// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
//
// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
// or visit www.oracle.com if you need additional information or have any
// questions.
//
//

// This file is automatically generated by running "m4 aarch64_sve_ad.m4". Do not edit ----

// AArch64 SVE Architecture Description File


// 4 bit signed offset -- for predicated load/store

operand vmemA_immIOffset4()
%{
  // (esize / msize) = 1
  predicate(Address::offset_ok_for_sve_immed(n->get_int(), 4,
            Matcher::scalable_vector_reg_size(T_BYTE)));
  match(ConI);

  op_cost(0);
  format %{ %}
  interface(CONST_INTER);
%}

operand vmemA_immLOffset4()
%{
  // (esize / msize) = 1
  predicate(Address::offset_ok_for_sve_immed(n->get_long(), 4,
            Matcher::scalable_vector_reg_size(T_BYTE)));
  match(ConL);

  op_cost(0);
  format %{ %}
  interface(CONST_INTER);
%}

operand vmemA_indOffI4(iRegP reg, vmemA_immIOffset4 off)
%{
  constraint(ALLOC_IN_RC(ptr_reg));
  match(AddP reg off);
  op_cost(0);
  format %{ "[$reg, $off]" %}
  interface(MEMORY_INTER) %{
    base($reg);
    index(0xffffffff);
    scale(0x0);
    disp($off);
  %}
%}

operand vmemA_indOffL4(iRegP reg, vmemA_immLOffset4 off)
%{
  constraint(ALLOC_IN_RC(ptr_reg));
  match(AddP reg off);
  op_cost(0);
  format %{ "[$reg, $off]" %}
  interface(MEMORY_INTER) %{
    base($reg);
    index(0xffffffff);
    scale(0x0);
    disp($off);
  %}
%}

// The indOff of vmemA is valid only when the vector element (load to/store from)
// size equals to memory element (load from/store to) size.
opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);

source_hpp %{
  bool op_sve_supported(int opcode, int vlen, BasicType bt);
  bool masked_op_sve_supported(int opcode, int vlen, BasicType bt);
%}

source %{

  typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
                                                             PRegister Pg, const Address &adr);

  // Predicated load/store, with optional ptrue to all elements of given predicate register.
  static void loadStoreA_predicated(C2_MacroAssembler masm, bool is_store, FloatRegister reg,
                                    PRegister pg, BasicType mem_elem_bt, BasicType vector_elem_bt,
                                    int opcode, Register base, int index, int size, int disp) {
    sve_mem_insn_predicate insn;
    int mesize = type2aelembytes(mem_elem_bt);
    if (index == -1) {
      assert(size == 0, "unsupported address mode: scale size = %d", size);
      switch(mesize) {
      case 1:
        insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
        break;
      case 2:
        insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
        break;
      case 4:
        insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
        break;
      case 8:
        insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
        break;
      default:
        assert(false, "unsupported");
        ShouldNotReachHere();
      }
      int imm4 = disp / mesize / Matcher::scalable_vector_reg_size(vector_elem_bt);
      (masm.*insn)(reg, Assembler::elemType_to_regVariant(vector_elem_bt), pg, Address(base, imm4));
    } else {
      assert(false, "unimplemented");
      ShouldNotReachHere();
    }
  }

  bool op_sve_supported(int opcode, int vlen, BasicType bt) {
    int length_in_bytes = vlen * type2aelembytes(bt);
    switch (opcode) {
      case Op_MulAddVS2VI:
      // No multiply reduction instructions
      case Op_MulReductionVD:
      case Op_MulReductionVF:
      case Op_MulReductionVI:
      case Op_MulReductionVL:
      // Others
      case Op_ExtractC:
      case Op_ExtractUB:
        return false;
      // Vector API specific
      case Op_VectorLoadShuffle:
      case Op_VectorRearrange:
        return vlen >= 4 && length_in_bytes <= MaxVectorSize;
      case Op_LoadVector:
      case Op_StoreVector:
        return Matcher::vector_size_supported(bt, vlen);
      case Op_ExpandV:
        if (UseSVE < 2 || is_subword_type(bt)) return false;
      case Op_VectorMaskToLong:
        if (vlen > 64) return false;
      default:
        break;
    }
    // By default, we only support vector operations with no less than 8 bytes and 2 elements.
    return 8 <= length_in_bytes && length_in_bytes <= MaxVectorSize && vlen >= 2;
  }

  bool masked_op_sve_supported(int opcode, int vlen, BasicType bt) {
    if (opcode == Op_VectorRearrange) {
      return false;
    }
    return op_sve_supported(opcode, vlen, bt);
  }
%}

definitions %{
  int_def SVE_COST             (200, 200);
%}


// All SVE instructions

// vector load/store

// Unpredicated vector load/store
instruct loadV(vReg dst, vmemA mem) %{
  predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16 &&
            n->as_LoadVector()->memory_size() == MaxVectorSize);
  match(Set dst (LoadVector mem));
  ins_cost(4 * SVE_COST);
  format %{ "sve_ldr $dst, $mem\t# vector (sve)" %}
  ins_encode %{
    FloatRegister dst_reg = as_FloatRegister($dst$$reg);
    BasicType bt = Matcher::vector_element_basic_type(this);
    loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
                          bt, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

instruct storeV(vReg src, vmemA mem) %{
  predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() >= 16 &&
            n->as_StoreVector()->memory_size() == MaxVectorSize);
  match(Set mem (StoreVector mem src));
  ins_cost(4 * SVE_COST);
  format %{ "sve_str $mem, $src\t# vector (sve)" %}
  ins_encode %{
    FloatRegister src_reg = as_FloatRegister($src$$reg);
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg, ptrue,
                          bt, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

// Load Vector (16 bits)
instruct loadV2_vreg(vReg dst, vmem2 mem)
%{
  predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 2);
  match(Set dst (LoadVector mem));
  ins_cost(4 * INSN_COST);
  format %{ "ldrh   $dst,$mem\t# vector (16 bits)" %}
  ins_encode( aarch64_enc_ldrvH(dst, mem) );
  ins_pipe(vload_reg_mem64);
%}

// Store Vector (16 bits)
instruct storeV2_vreg(vReg src, vmem2 mem)
%{
  predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 2);
  match(Set mem (StoreVector mem src));
  ins_cost(4 * INSN_COST);
  format %{ "strh   $mem,$src\t# vector (16 bits)" %}
  ins_encode( aarch64_enc_strvH(src, mem) );
  ins_pipe(vstore_reg_mem64);
%}

// Load Vector (32 bits)
instruct loadV4_vreg(vReg dst, vmem4 mem)
%{
  predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 4);
  match(Set dst (LoadVector mem));
  ins_cost(4 * INSN_COST);
  format %{ "ldrs   $dst,$mem\t# vector (32 bits)" %}
  ins_encode( aarch64_enc_ldrvS(dst, mem) );
  ins_pipe(vload_reg_mem64);
%}

// Store Vector (32 bits)
instruct storeV4_vreg(vReg src, vmem4 mem)
%{
  predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 4);
  match(Set mem (StoreVector mem src));
  ins_cost(4 * INSN_COST);
  format %{ "strs   $mem,$src\t# vector (32 bits)" %}
  ins_encode( aarch64_enc_strvS(src, mem) );
  ins_pipe(vstore_reg_mem64);
%}

// Load Vector (64 bits)
instruct loadV8_vreg(vReg dst, vmem8 mem)
%{
  predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 8);
  match(Set dst (LoadVector mem));
  ins_cost(4 * INSN_COST);
  format %{ "ldrd   $dst,$mem\t# vector (64 bits)" %}
  ins_encode( aarch64_enc_ldrvD(dst, mem) );
  ins_pipe(vload_reg_mem64);
%}

// Store Vector (64 bits)
instruct storeV8_vreg(vReg src, vmem8 mem)
%{
  predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 8);
  match(Set mem (StoreVector mem src));
  ins_cost(4 * INSN_COST);
  format %{ "strd   $mem,$src\t# vector (64 bits)" %}
  ins_encode( aarch64_enc_strvD(src, mem) );
  ins_pipe(vstore_reg_mem64);
%}

// Load Vector (128 bits)
instruct loadV16_vreg(vReg dst, vmem16 mem)
%{
  predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() == 16);
  match(Set dst (LoadVector mem));
  ins_cost(4 * INSN_COST);
  format %{ "ldrq   $dst,$mem\t# vector (128 bits)" %}
  ins_encode( aarch64_enc_ldrvQ(dst, mem) );
  ins_pipe(vload_reg_mem128);
%}

// Store Vector (128 bits)
instruct storeV16_vreg(vReg src, vmem16 mem)
%{
  predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() == 16);
  match(Set mem (StoreVector mem src));
  ins_cost(4 * INSN_COST);
  format %{ "strq   $mem,$src\t# vector (128 bits)" %}
  ins_encode( aarch64_enc_strvQ(src, mem) );
  ins_pipe(vstore_reg_mem128);
%}

// Predicated vector load/store, based on the vector length of the node.
// Only load/store values in the range of the memory_size. This is needed
// when the memory_size is lower than the hardware supported max vector size.
// And this might happen for Vector API mask vector load/store.
instruct loadV_partial(vReg dst, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() > 16 &&
            n->as_LoadVector()->memory_size() < MaxVectorSize);
  match(Set dst (LoadVector mem));
  effect(TEMP pgtmp, KILL cr);
  ins_cost(6 * SVE_COST);
  format %{ "sve_ptrue $pgtmp, vector_length\n\t"
            "sve_ldr $dst, $pgtmp, $mem\t# load vector partial" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
                         Matcher::vector_length(this));
    FloatRegister dst_reg = as_FloatRegister($dst$$reg);
    loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, dst_reg,
                          as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

instruct storeV_partial(vReg src, vmemA mem, pRegGov pgtmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() > 16 &&
            n->as_StoreVector()->memory_size() < MaxVectorSize);
  match(Set mem (StoreVector mem src));
  effect(TEMP pgtmp, KILL cr);
  ins_cost(5 * SVE_COST);
  format %{ "sve_ptrue $pgtmp, vector_length\n\t"
            "sve_str $src, $pgtmp, $mem\t# store vector partial" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ elemType_to_regVariant(bt),
                         Matcher::vector_length(this, $src));
    FloatRegister src_reg = as_FloatRegister($src$$reg);
    loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, src_reg,
                          as_PRegister($pgtmp$$reg), bt, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

// vector load/store - predicated

instruct loadV_masked(vReg dst, vmemA mem, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst (LoadVectorMasked mem pg));
  ins_cost(4 * SVE_COST);
  format %{ "sve_ldr $dst, $pg, $mem\t# load vector predicated (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($dst$$reg),
                          as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

instruct storeV_masked(vReg src, vmemA mem, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set mem (StoreVectorMasked mem (Binary src pg)));
  ins_cost(4 * SVE_COST);
  format %{ "sve_str $mem, $pg, $src\t# store vector predicated (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($src$$reg),
                          as_PRegister($pg$$reg), bt, bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

// mask logical and/or/xor

instruct vmask_and(pRegGov pd, pRegGov pn, pRegGov pm) %{
  predicate(UseSVE > 0);
  match(Set pd (AndVMask pn pm));
  ins_cost(SVE_COST);
  format %{ "sve_and $pd, $pn, $pm\t# predicate (sve)" %}
  ins_encode %{
    __ sve_and(as_PRegister($pd$$reg), ptrue,
               as_PRegister($pn$$reg), as_PRegister($pm$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_or(pRegGov pd, pRegGov pn, pRegGov pm) %{
  predicate(UseSVE > 0);
  match(Set pd (OrVMask pn pm));
  ins_cost(SVE_COST);
  format %{ "sve_orr $pd, $pn, $pm\t# predicate (sve)" %}
  ins_encode %{
    __ sve_orr(as_PRegister($pd$$reg), ptrue,
               as_PRegister($pn$$reg), as_PRegister($pm$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_xor(pRegGov pd, pRegGov pn, pRegGov pm) %{
  predicate(UseSVE > 0);
  match(Set pd (XorVMask pn pm));
  ins_cost(SVE_COST);
  format %{ "sve_eor $pd, $pn, $pm\t# predicate (sve)" %}
  ins_encode %{
    __ sve_eor(as_PRegister($pd$$reg), ptrue,
               as_PRegister($pn$$reg), as_PRegister($pm$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// mask logical and_not

instruct vmask_and_notI(pRegGov pd, pRegGov pn, pRegGov pm, immI_M1 m1) %{
  predicate(UseSVE > 0);
  match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
  ins_cost(SVE_COST);
  format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (B/H/S)" %}
  ins_encode %{
    __ sve_bic(as_PRegister($pd$$reg), ptrue,
               as_PRegister($pn$$reg), as_PRegister($pm$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_and_notL(pRegGov pd, pRegGov pn, pRegGov pm, immL_M1 m1) %{
  predicate(UseSVE > 0);
  match(Set pd (AndVMask pn (XorVMask pm (MaskAll m1))));
  ins_cost(SVE_COST);
  format %{ "sve_bic $pd, $pn, $pm\t# predciate (sve) (D)" %}
  ins_encode %{
    __ sve_bic(as_PRegister($pd$$reg), ptrue,
               as_PRegister($pn$$reg), as_PRegister($pm$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector reinterpret

instruct reinterpret(vReg dst) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() ==
                          n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src == dst
  match(Set dst (VectorReinterpret dst));
  ins_cost(0);
  format %{ "# reinterpret $dst\t# do nothing" %}
  ins_encode %{
    // empty
  %}
  ins_pipe(pipe_class_empty);
%}

instruct reinterpretResize(vReg dst, vReg src, pRegGov pgtmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() !=
                          n->in(1)->bottom_type()->is_vect()->length_in_bytes());  // src != dst
  match(Set dst (VectorReinterpret src));
  effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
  ins_cost(3 * SVE_COST);
  format %{ "reinterpretResize $dst, $src\t# vector (sve)" %}
  ins_encode %{
    uint length_in_bytes_src = Matcher::vector_length_in_bytes(this, $src);
    uint length_in_bytes_dst = Matcher::vector_length_in_bytes(this);
    uint length_in_bytes_resize = length_in_bytes_src < length_in_bytes_dst ?
                                  length_in_bytes_src : length_in_bytes_dst;
    assert(length_in_bytes_src <= MaxVectorSize && length_in_bytes_dst <= MaxVectorSize,
           "invalid vector length");
    __ sve_ptrue_lanecnt(as_PRegister($pgtmp$$reg), __ B, length_in_bytes_resize);
    __ sve_dup(as_FloatRegister($dst$$reg), __ B, 0);
    __ sve_sel(as_FloatRegister($dst$$reg), __ B, as_PRegister($pgtmp$$reg),
               as_FloatRegister($src$$reg), as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector mask reinterpret

instruct vmask_reinterpret_same_esize(pRegGov dst_src) %{
  predicate(UseSVE > 0 &&
            n->as_Vector()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
            n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
  match(Set dst_src (VectorReinterpret dst_src));
  ins_cost(0);
  format %{ "# vmask_reinterpret $dst_src\t# do nothing" %}
  ins_encode %{
    // empty
  %}
  ins_pipe(pipe_class_empty);
%}

instruct vmask_reinterpret_diff_esize(pRegGov dst, pRegGov src, vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->as_Vector()->length() != n->in(1)->bottom_type()->is_vect()->length() &&
            n->as_Vector()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
  match(Set dst (VectorReinterpret src));
  effect(TEMP tmp, KILL cr);
  ins_cost(2 * SVE_COST);
  format %{ "# vmask_reinterpret $dst, $src\t# vector (sve)" %}
  ins_encode %{
    BasicType from_bt = Matcher::vector_element_basic_type(this, $src);
    Assembler::SIMD_RegVariant from_size = __ elemType_to_regVariant(from_bt);
    BasicType to_bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
    __ sve_cpy(as_FloatRegister($tmp$$reg), from_size, as_PRegister($src$$reg), -1, false);
    __ sve_cmp(Assembler::EQ, as_PRegister($dst$$reg), to_size, ptrue, as_FloatRegister($tmp$$reg), -1);
  %}
  ins_pipe(pipe_slow);
%}

// vector abs

instruct vabsB(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (AbsVB src));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst, $src\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst$$reg), __ B,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsS(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (AbsVS src));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst, $src\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst$$reg), __ H,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsI(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (AbsVI src));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsL(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (AbsVL src));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsF(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (AbsVF src));
  ins_cost(SVE_COST);
  format %{ "sve_fabs $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fabs(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsD(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (AbsVD src));
  ins_cost(SVE_COST);
  format %{ "sve_fabs $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fabs(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector abs - predicated

instruct vabsB_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AbsVB dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst_src$$reg), __ B,
            as_PRegister($pg$$reg),
            as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsS_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AbsVS dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst_src$$reg), __ H,
            as_PRegister($pg$$reg),
            as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsI_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AbsVI dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst_src$$reg), __ S,
            as_PRegister($pg$$reg),
            as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsL_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AbsVL dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst_src$$reg), __ D,
            as_PRegister($pg$$reg),
            as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsF_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AbsVF dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fabs(as_FloatRegister($dst_src$$reg), __ S,
            as_PRegister($pg$$reg),
            as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsD_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AbsVD dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_fabs $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fabs(as_FloatRegister($dst_src$$reg), __ D,
            as_PRegister($pg$$reg),
            as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector add

instruct vaddB(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (AddVB src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst$$reg), __ B,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddS(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (AddVS src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst$$reg), __ H,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddI(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (AddVI src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddL(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (AddVL src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddF(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (AddVF src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fadd(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddD(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (AddVD src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fadd(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector add - predicated

instruct vaddB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVB (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst_src1$$reg), __ B,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVS (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst_src1$$reg), __ H,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVI (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst_src1$$reg), __ S,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVL (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst_src1$$reg), __ D,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVF (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ S,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVD (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_fadd $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fadd(as_FloatRegister($dst_src1$$reg), __ D,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector add reg imm (unpredicated)

instruct vaddImmB(vReg dst_src, immBAddSubV con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AddVB dst_src (ReplicateB con)));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
  ins_encode %{
    int32_t val = $con$$constant;
    if (val > 0){
      __ sve_add(as_FloatRegister($dst_src$$reg), __ B, val);
    } else if (val < 0){
      __ sve_sub(as_FloatRegister($dst_src$$reg), __ B, -val);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddImmS(vReg dst_src, immIAddSubV con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AddVS dst_src (ReplicateS con)));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
  ins_encode %{
    int32_t val = $con$$constant;
    if (val > 0){
      __ sve_add(as_FloatRegister($dst_src$$reg), __ H, val);
    } else if (val < 0){
      __ sve_sub(as_FloatRegister($dst_src$$reg), __ H, -val);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddImmI(vReg dst_src, immIAddSubV con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AddVI dst_src (ReplicateI con)));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
  ins_encode %{
    int32_t val = $con$$constant;
    if (val > 0){
      __ sve_add(as_FloatRegister($dst_src$$reg), __ S, val);
    } else if (val < 0){
      __ sve_sub(as_FloatRegister($dst_src$$reg), __ S, -val);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddImmL(vReg dst_src, immLAddSubV con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AddVL dst_src (ReplicateL con)));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
  ins_encode %{
    int32_t val = $con$$constant;
    if (val > 0){
      __ sve_add(as_FloatRegister($dst_src$$reg), __ D, val);
    } else if (val < 0){
      __ sve_sub(as_FloatRegister($dst_src$$reg), __ D, -val);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector binary op reg imm (unpredicated)

instruct vandB(vReg dst_src, immBLog con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AndV dst_src (ReplicateB con)));
  ins_cost(SVE_COST);
  format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_and(as_FloatRegister($dst_src$$reg), __ B,
         (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vandH(vReg dst_src, immSLog con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AndV dst_src (ReplicateS con)));
  ins_cost(SVE_COST);
  format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_and(as_FloatRegister($dst_src$$reg), __ H,
         (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vandS(vReg dst_src, immILog con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AndV dst_src (ReplicateI con)));
  ins_cost(SVE_COST);
  format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_and(as_FloatRegister($dst_src$$reg), __ S,
         (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vandD(vReg dst_src, immLLog con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (AndV dst_src (ReplicateL con)));
  ins_cost(SVE_COST);
  format %{ "sve_and $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_and(as_FloatRegister($dst_src$$reg), __ D,
         (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vorB(vReg dst_src, immBLog con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (OrV dst_src (ReplicateB con)));
  ins_cost(SVE_COST);
  format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_orr(as_FloatRegister($dst_src$$reg), __ B,
         (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vorH(vReg dst_src, immSLog con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (OrV dst_src (ReplicateS con)));
  ins_cost(SVE_COST);
  format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_orr(as_FloatRegister($dst_src$$reg), __ H,
         (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vorS(vReg dst_src, immILog con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (OrV dst_src (ReplicateI con)));
  ins_cost(SVE_COST);
  format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_orr(as_FloatRegister($dst_src$$reg), __ S,
         (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vorD(vReg dst_src, immLLog con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (OrV dst_src (ReplicateL con)));
  ins_cost(SVE_COST);
  format %{ "sve_orr $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_orr(as_FloatRegister($dst_src$$reg), __ D,
         (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vxorB(vReg dst_src, immBLog con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (XorV dst_src (ReplicateB con)));
  ins_cost(SVE_COST);
  format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_eor(as_FloatRegister($dst_src$$reg), __ B,
         (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vxorH(vReg dst_src, immSLog con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (XorV dst_src (ReplicateS con)));
  ins_cost(SVE_COST);
  format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_eor(as_FloatRegister($dst_src$$reg), __ H,
         (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vxorS(vReg dst_src, immILog con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (XorV dst_src (ReplicateI con)));
  ins_cost(SVE_COST);
  format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_eor(as_FloatRegister($dst_src$$reg), __ S,
         (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}

instruct vxorD(vReg dst_src, immLLog con) %{
  predicate(UseSVE > 0);
  match(Set dst_src (XorV dst_src (ReplicateL con)));
  ins_cost(SVE_COST);
  format %{ "sve_eor $dst_src, $dst_src, $con\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_eor(as_FloatRegister($dst_src$$reg), __ D,
         (uint64_t)($con$$constant));
  %}
  ins_pipe(pipe_slow);
%}
// vector and

instruct vand(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (AndV src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_and  $dst, $src1, $src2\t# vector (sve)" %}
  ins_encode %{
    __ sve_and(as_FloatRegister($dst$$reg),
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector or

instruct vor(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (OrV src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_orr  $dst, $src1, $src2\t# vector (sve)" %}
  ins_encode %{
    __ sve_orr(as_FloatRegister($dst$$reg),
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector xor

instruct vxor(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (XorV src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_eor  $dst, $src1, $src2\t# vector (sve)" %}
  ins_encode %{
    __ sve_eor(as_FloatRegister($dst$$reg),
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector and - predicated

instruct vand_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AndV (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_and $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_and(as_FloatRegister($dst_src1$$reg), size,
          as_PRegister($pg$$reg),
          as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector or - predicated

instruct vor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (OrV (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_orr $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_orr(as_FloatRegister($dst_src1$$reg), size,
          as_PRegister($pg$$reg),
          as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector xor - predicated

instruct vxor_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (XorV (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_eor $dst_src1, $pg, $dst_src1, $src2\t # vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_eor(as_FloatRegister($dst_src1$$reg), size,
          as_PRegister($pg$$reg),
          as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector not

instruct vnotI(vReg dst, vReg src, immI_M1 m1) %{
  predicate(UseSVE > 0);
  match(Set dst (XorV src (ReplicateB m1)));
  match(Set dst (XorV src (ReplicateS m1)));
  match(Set dst (XorV src (ReplicateI m1)));
  ins_cost(SVE_COST);
  format %{ "sve_not $dst, $src\t# vector (sve) B/H/S" %}
  ins_encode %{
    __ sve_not(as_FloatRegister($dst$$reg), __ D,
               ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{
  predicate(UseSVE > 0);
  match(Set dst (XorV src (ReplicateL m1)));
  ins_cost(SVE_COST);
  format %{ "sve_not $dst, $src\t# vector (sve) D" %}
  ins_encode %{
    __ sve_not(as_FloatRegister($dst$$reg), __ D,
               ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector not - predicated

instruct vnotI_masked(vReg dst_src, immI_M1 m1, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (XorV (Binary dst_src (ReplicateB m1)) pg));
  match(Set dst_src (XorV (Binary dst_src (ReplicateS m1)) pg));
  match(Set dst_src (XorV (Binary dst_src (ReplicateI m1)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_not $dst_src, $pg, $dst_src\t# vector (sve) B/H/S" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_not(as_FloatRegister($dst_src$$reg), __ elemType_to_regVariant(bt),
               as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vnotL_masked(vReg dst_src, immL_M1 m1, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (XorV (Binary dst_src (ReplicateL m1)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_not $dst_src, $pg, $dst_src\t# vector (sve) D" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_not(as_FloatRegister($dst_src$$reg), __ elemType_to_regVariant(bt),
               as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector and_not

instruct vand_notI(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
  predicate(UseSVE > 0);
  match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
  match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
  match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));
  ins_cost(SVE_COST);
  format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) B/H/S" %}
  ins_encode %{
    __ sve_bic(as_FloatRegister($dst$$reg),
               as_FloatRegister($src1$$reg),
               as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vand_notL(vReg dst, vReg src1, vReg src2, immL_M1 m1) %{
  predicate(UseSVE > 0);
  match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));
  ins_cost(SVE_COST);
  format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) D" %}
  ins_encode %{
    __ sve_bic(as_FloatRegister($dst$$reg),
               as_FloatRegister($src1$$reg),
               as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector and_not - predicated

instruct vand_notI_masked(vReg dst_src1, vReg src2, immI_M1 m1, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateB m1))) pg));
  match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateS m1))) pg));
  match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateI m1))) pg));
  ins_cost(SVE_COST);
  format %{ "sve_bic $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) B/H/S" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_bic(as_FloatRegister($dst_src1$$reg), __ elemType_to_regVariant(bt),
               as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vand_notL_masked(vReg dst_src1, vReg src2, immL_M1 m1, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AndV (Binary dst_src1 (XorV src2 (ReplicateL m1))) pg));
  ins_cost(SVE_COST);
  format %{ "sve_bic $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) D" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_bic(as_FloatRegister($dst_src1$$reg), __ elemType_to_regVariant(bt),
               as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector float div

instruct vdivF(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (DivVF dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vdivD(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (DivVD dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector float div - predicated

instruct vfdivF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (DivVF (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vfdivD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (DivVD (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_fdiv $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector min/max

instruct vmin(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MinV dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_min $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    if (is_floating_point_type(bt)) {
      __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
                  ptrue, as_FloatRegister($src2$$reg));
    } else {
      assert(is_integral_type(bt), "unsupported type");
      __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
                  ptrue, as_FloatRegister($src2$$reg));
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmax(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MaxV dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_max $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    if (is_floating_point_type(bt)) {
      __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
                  ptrue, as_FloatRegister($src2$$reg));
    } else {
      assert(is_integral_type(bt), "unsupported type");
      __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
                  ptrue, as_FloatRegister($src2$$reg));
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector min/max - predicated

instruct vmin_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MinV (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_min $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    if (is_floating_point_type(bt)) {
      __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
                  as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
    } else {
      assert(is_integral_type(bt), "unsupported type");
      __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
                  as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmax_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MaxV (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_max $dst_src1, $pg, $dst_src1, $src2\t# vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    if (is_floating_point_type(bt)) {
      __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
                  as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
    } else {
      assert(is_integral_type(bt), "unsupported type");
      __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
                  as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector fmla

// dst_src1 = dst_src1 + src2 * src3
instruct vfmlaF(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0);
  match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * src3
instruct vfmlaD(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0);
  match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector fmad - predicated

// dst_src1 = dst_src1 * src2 + src3
instruct vfmadF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseFMA && UseSVE > 0);
  match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary src3 pg)));
  ins_cost(SVE_COST);
  format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 * src2 + src3
instruct vfmadD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseFMA && UseSVE > 0);
  match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary src3 pg)));
  ins_cost(SVE_COST);
  format %{ "sve_fmad $dst_src1, $pg, $src2, $src3\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fmad(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector fmls

// dst_src1 = dst_src1 + -src2 * src3
// The NegVF must not be predicated.
instruct vfmlsF1(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVF dst_src1 (Binary (NegVF src2) src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * -src3
// The NegVF must not be predicated.
instruct vfmlsF2(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
  ins_cost(SVE_COST);
  format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + -src2 * src3
// The NegVD must not be predicated.
instruct vfmlsD1(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVD dst_src1 (Binary (NegVD src2) src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * -src3
// The NegVD must not be predicated.
instruct vfmlsD2(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3))));
  ins_cost(SVE_COST);
  format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector fmsb - predicated

// dst_src1 = dst_src1 * -src2 + src3
// The NegVF must not be predicated.
instruct vfmsbF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(1)->in(2)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary src3 pg)));
  ins_cost(SVE_COST);
  format %{ "sve_fmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fmsb(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 * -src2 + src3
// The NegVD must not be predicated.
instruct vfmsbD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(1)->in(2)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary src3 pg)));
  ins_cost(SVE_COST);
  format %{ "sve_fmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fmsb(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector fnmla

// dst_src1 = -dst_src1 + -src2 * src3
// The NegVF must not be predicated.
instruct vfnmlaF1(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(1)->as_Vector()->is_predicated_vector() &&
            !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary (NegVF src2) src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = -dst_src1 + src2 * -src3
// The NegVF must not be predicated.
instruct vfnmlaF2(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(1)->as_Vector()->is_predicated_vector() &&
            !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3))));
  ins_cost(SVE_COST);
  format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = -dst_src1 + -src2 * src3
// The NegVD must not be predicated.
instruct vfnmlaD1(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(1)->as_Vector()->is_predicated_vector() &&
            !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary (NegVD src2) src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = -dst_src1 + src2 * -src3
// The NegVD must not be predicated.
instruct vfnmlaD2(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(1)->as_Vector()->is_predicated_vector() &&
            !n->in(2)->in(2)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3))));
  ins_cost(SVE_COST);
  format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector fnmad - predicated

// dst_src1 = -src3 + dst_src1 * -src2
// The NegVF must not be predicated.
instruct vfnmadF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(1)->in(2)->as_Vector()->is_predicated_vector() &&
            !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVF (Binary dst_src1 (NegVF src2)) (Binary (NegVF src3) pg)));
  ins_cost(SVE_COST);
  format %{ "sve_fnmad $dst_src1, $pg, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fnmad(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = -src3 + dst_src1 * -src2
// The NegVD must not be predicated.
instruct vfnmadD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(1)->in(2)->as_Vector()->is_predicated_vector() &&
            !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVD (Binary dst_src1 (NegVD src2)) (Binary (NegVD src3) pg)));
  ins_cost(SVE_COST);
  format %{ "sve_fnmad $dst_src1, $pg, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fnmad(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector fnmls

// dst_src1 = -dst_src1 + src2 * src3
// The NegVF must not be predicated.
instruct vfnmlsF(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(1)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = -dst_src1 + src2 * src3
// The NegVD must not be predicated.
instruct vfnmlsD(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(1)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector fnmsb - predicated

// dst_src1 = -src3 + dst_src1 * src2
// The NegVF must not be predicated.
instruct vfnmsbF_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVF (Binary dst_src1 src2) (Binary (NegVF src3) pg)));
  ins_cost(SVE_COST);
  format %{ "sve_fnmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fnmsb(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = -src3 + dst_src1 * src2
// The NegVD must not be predicated.
instruct vfnmsbD_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg) %{
  predicate(UseFMA && UseSVE > 0 &&
            !n->in(2)->in(1)->as_Vector()->is_predicated_vector());
  match(Set dst_src1 (FmaVD (Binary dst_src1 src2) (Binary (NegVD src3) pg)));
  ins_cost(SVE_COST);
  format %{ "sve_fnmsb $dst_src1, $pg, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fnmsb(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector mla

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaB(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVB dst_src1 (MulVB src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ B,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaS(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVS dst_src1 (MulVS src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ H,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaI(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVI dst_src1 (MulVI src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ S,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaL(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVL dst_src1 (MulVL src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ D,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector mla - predicated

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaB_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVB (Binary dst_src1 (MulVB src2 src3)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ B, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaS_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVS (Binary dst_src1 (MulVS src2 src3)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ H, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaI_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVI (Binary dst_src1 (MulVI src2 src3)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaL_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (AddVL (Binary dst_src1 (MulVL src2 src3)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, $pg, src2, src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector mls

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsB(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVB dst_src1 (MulVB src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ B,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsS(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVS dst_src1 (MulVS src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ H,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsI(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVI dst_src1 (MulVI src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsL(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector mls - predicated

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsB_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVB (Binary dst_src1 (MulVB src2 src3)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ B, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsS_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVS (Binary dst_src1 (MulVS src2 src3)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ H, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsI_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVI (Binary dst_src1 (MulVI src2 src3)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsL_masked(vReg dst_src1, vReg src2, vReg src3, pRegGov pg)
%{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVL (Binary dst_src1 (MulVL src2 src3)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, $pg, src2, src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D, as_PRegister($pg$$reg),
         as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector mul

instruct vmulB(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVB dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulS(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVS dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulI(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVI dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulL(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVL dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulF(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (MulVF src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fmul(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulD(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (MulVD src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fmul(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector mul - predicated

instruct vmulB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVB (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVS (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVI (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVL (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVF (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ S,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (MulVD (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_fmul $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fmul(as_FloatRegister($dst_src1$$reg), __ D,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector neg

instruct vnegI(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (NegVI src));
  ins_cost(SVE_COST);
  format %{ "sve_neg $dst, $src\t# vector (sve) (B/H/S)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_neg(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vnegL(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (NegVL src));
  ins_cost(SVE_COST);
  format %{ "sve_neg $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_neg(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vnegF(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (NegVF src));
  ins_cost(SVE_COST);
  format %{ "sve_fneg $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fneg(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vnegD(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (NegVD src));
  ins_cost(SVE_COST);
  format %{ "sve_fneg $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fneg(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector neg - predicated

instruct vnegI_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (NegVI dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_neg $dst_src, $pg, $dst_src\t# vector (sve) (B/H/S)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_neg(as_FloatRegister($dst_src$$reg), __ elemType_to_regVariant(bt),
            as_PRegister($pg$$reg),
            as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vnegL_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (NegVL dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_neg $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_neg(as_FloatRegister($dst_src$$reg), __ D,
            as_PRegister($pg$$reg),
            as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vnegF_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (NegVF dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fneg(as_FloatRegister($dst_src$$reg), __ S,
            as_PRegister($pg$$reg),
            as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vnegD_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (NegVD dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_fneg $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fneg(as_FloatRegister($dst_src$$reg), __ D,
            as_PRegister($pg$$reg),
            as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector popcount

instruct vpopcountI(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (PopCountVI src));
  ins_cost(SVE_COST);
  format %{ "sve_cnt $dst, $src\t# vector (sve) (B/H/S)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_cnt(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vpopcountL(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector() &&
            n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (PopCountVL src));
  ins_cost(SVE_COST);
  format %{ "sve_cnt $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_cnt(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// If the PopCountVL is generated by auto-vectorization, the dst basic
// type is T_INT. And once we have unified the type definition for
// Vector API and auto-vectorization, this rule can be merged with
// "vpopcountL" rule.
instruct vpopcountLI(vReg dst, vReg src, vReg vtmp) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector() &&
            n->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (PopCountVL src));
  effect(TEMP_DEF dst, TEMP vtmp);
  ins_cost(3 * SVE_COST);
  format %{ "sve_cnt $dst, $src\n\t"
            "sve_dup $vtmp, #0\n\t"
            "sve_uzp1 $dst, $dst, $vtmp\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_cnt(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
    __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($dst$$reg), __ D, as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector popcount - predicated

instruct vpopcountI_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (PopCountVI dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_cnt $dst_src, $pg, $dst_src\t# vector (sve) (B/H/S)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_cnt(as_FloatRegister($dst_src$$reg), __ elemType_to_regVariant(bt),
         as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vpopcountL_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst_src (PopCountVL dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_cnt $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_cnt(as_FloatRegister($dst_src$$reg), __ D,
         as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector blend

instruct vblend(vReg dst, vReg src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorBlend (Binary src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_sel $dst, $pg, $src2, $src1\t# vector blend (sve)" %}
  ins_encode %{
    Assembler::SIMD_RegVariant size =
               __ elemType_to_regVariant(Matcher::vector_element_basic_type(this));
    __ sve_sel(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
               as_FloatRegister($src2$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector store mask

instruct vstoremaskB(vReg dst, pRegGov src, immI_1 size) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorStoreMask src size));
  ins_cost(SVE_COST);
  format %{ "vstoremask $dst, $src\t# vector store mask (sve) (B)" %}
  ins_encode %{
    __ sve_cpy(as_FloatRegister($dst$$reg), __ B, as_PRegister($src$$reg), 1, false);
  %}
  ins_pipe(pipe_slow);
%}

instruct vstoremask_narrow(vReg dst, pRegGov src, vReg tmp, immI_gt_1 size) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorStoreMask src size));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(3 * SVE_COST);
  format %{ "vstoremask $dst, $src\t# vector store mask (sve) (H/S/D)" %}
  ins_encode %{
    Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant((int)$size$$constant);
    __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($src$$reg), 1, false);
    __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
                         as_FloatRegister($dst$$reg), size, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// Combine LoadVector+VectorLoadMask when the vector element type is not T_BYTE

instruct vloadmask_loadV(pRegGov dst, indirect mem, vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->as_Vector()->length_in_bytes() == MaxVectorSize &&
            type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
  match(Set dst (VectorLoadMask (LoadVector mem)));
  effect(TEMP tmp, KILL cr);
  ins_cost(3 * SVE_COST);
  format %{ "sve_ld1b $tmp, $mem\n\t"
            "sve_cmpne $dst, $tmp, 0\t# load vector mask (sve) (H/S/D)" %}
  ins_encode %{
    // Load mask values which are boolean type, and extend them to the
    // expected vector element type. Convert the vector to predicate.
    BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
    loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($tmp$$reg),
                          ptrue, T_BOOLEAN, to_vect_bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
    __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ elemType_to_regVariant(to_vect_bt),
               ptrue, as_FloatRegister($tmp$$reg), 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct vloadmask_loadV_partial(pRegGov dst, indirect mem, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->as_Vector()->length_in_bytes() > 16 &&
            n->as_Vector()->length_in_bytes() < MaxVectorSize &&
            type2aelembytes(n->bottom_type()->is_vect()->element_basic_type()) > 1);
  match(Set dst (VectorLoadMask (LoadVector mem)));
  effect(TEMP vtmp, TEMP ptmp, KILL cr);
  ins_cost(6 * SVE_COST);
  format %{ "vloadmask_loadV $dst, $mem\t# load vector mask partial (sve) (H/S/D)" %}
  ins_encode %{
    // Load valid mask values which are boolean type, and extend them to the
    // expected vector element type. Convert the vector to predicate.
    BasicType to_vect_bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(to_vect_bt);
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this));
    loadStoreA_predicated(C2_MacroAssembler(&cbuf), false, as_FloatRegister($vtmp$$reg),
                          as_PRegister($ptmp$$reg), T_BOOLEAN, to_vect_bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
    __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($vtmp$$reg), 0);
  %}
  ins_pipe(pipe_slow);
%}

// Combine VectorStoreMask+StoreVector when the vector element type is not T_BYTE

instruct storeV_vstoremask(indirect mem, pRegGov src, vReg tmp, immI_gt_1 esize) %{
  predicate(UseSVE > 0 &&
            Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) == MaxVectorSize);
  match(Set mem (StoreVector mem (VectorStoreMask src esize)));
  effect(TEMP tmp);
  ins_cost(3 * SVE_COST);
  format %{ "sve_cpy $tmp, $src, 1\n\t"
            "sve_st1b $tmp, $mem\t# store vector mask (sve) (H/S/D)" %}
  ins_encode %{
    BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
    assert(type2aelembytes(from_vect_bt) == (int)$esize$$constant, "unsupported type.");
    Assembler::SIMD_RegVariant size = __ elemBytes_to_regVariant($esize$$constant);
    __ sve_cpy(as_FloatRegister($tmp$$reg), size, as_PRegister($src$$reg), 1, false);
    loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($tmp$$reg),
                          ptrue, T_BOOLEAN, from_vect_bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

instruct storeV_vstoremask_partial(indirect mem, pRegGov src, vReg vtmp,
                                   immI_gt_1 esize, pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->as_StoreVector()->memory_size() > 16 &&
            type2aelembytes(n->as_StoreVector()->vect_type()->element_basic_type()) > 1 &&
            Matcher::vector_length_in_bytes(n->as_StoreVector()->in(MemNode::ValueIn)->in(1)) < MaxVectorSize);
  match(Set mem (StoreVector mem (VectorStoreMask src esize)));
  effect(TEMP vtmp, TEMP ptmp, KILL cr);
  format %{ "storeV_vstoremask $src, $mem\t# store vector mask partial (sve) (H/S/D)" %}
  ins_cost(6 * SVE_COST);
  ins_encode %{
    // Convert the valid src predicate to vector, and store the vector
    // elements as boolean values.
    BasicType from_vect_bt = Matcher::vector_element_basic_type(this, $src);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(from_vect_bt);
    __ sve_cpy(as_FloatRegister($vtmp$$reg), size, as_PRegister($src$$reg), 1, false);
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
    loadStoreA_predicated(C2_MacroAssembler(&cbuf), true, as_FloatRegister($vtmp$$reg),
                          as_PRegister($ptmp$$reg), T_BOOLEAN, from_vect_bt, $mem->opcode(),
                          as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

// vector add reduction

instruct reduce_addI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
  match(Set dst (AddReductionVI src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction (sve) (may extend)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           ptrue, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
  match(Set dst (AddReductionVL src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction (sve)" %}
  ins_encode %{
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           ptrue, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addF(vRegF src1_dst, vReg src2) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
  match(Set src1_dst (AddReductionVF src1_dst src2));
  ins_cost(SVE_COST);
  format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addD(vRegD src1_dst, vReg src2) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
  match(Set src1_dst (AddReductionVD src1_dst src2));
  ins_cost(SVE_COST);
  format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
  match(Set dst (AddReductionVI src1 src2));
  effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
  ins_cost(2 * SVE_COST);
  format %{ "sve_reduce_addI $dst, $src1, $src2\t# addI reduction partial (sve) (may extend)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
  match(Set dst (AddReductionVL src1 src2));
  effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
  ins_cost(2 * SVE_COST);
  format %{ "sve_reduce_addL $dst, $src1, $src2\t# addL reduction partial (sve)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addF_partial(vRegF src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
  match(Set src1_dst (AddReductionVF src1_dst src2));
  ins_cost(SVE_COST);
  effect(TEMP ptmp, KILL cr);
  format %{ "sve_reduce_addF $src1_dst, $src1_dst, $src2\t# addF reduction partial (sve) (S)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
    __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
                 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addD_partial(vRegD src1_dst, vReg src2, pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
  match(Set src1_dst (AddReductionVD src1_dst src2));
  ins_cost(SVE_COST);
  effect(TEMP ptmp, KILL cr);
  format %{ "sve_reduce_addD $src1_dst, $src1_dst, $src2\t# addD reduction partial (sve) (D)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
    __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
                 as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector add reduction - predicated

instruct reduce_addI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst (AddReductionVI (Binary src1 src2) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_addI $dst, $src1, $pg, $src2\t# addI reduction predicated (sve) (may extend)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst (AddReductionVL (Binary src1 src2) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_addL $dst, $src1, $pg, $src2\t# addL reduction predicated (sve)" %}
  ins_encode %{
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addF_masked(vRegF src1_dst, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set src1_dst (AddReductionVF (Binary src1_dst src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_reduce_addF $src1_dst, $pg, $src2\t# addF reduction predicated (sve)" %}
  ins_encode %{
    __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
                 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addD_masked(vRegD src1_dst, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set src1_dst (AddReductionVD (Binary src1_dst src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_reduce_addD $src1_dst, $pg, $src2\t# addD reduction predicated (sve)" %}
  ins_encode %{
    __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
                 as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector and reduction

instruct reduce_andI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
  match(Set dst (AndReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction (sve) (may extend)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           ptrue, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_andL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
  match(Set dst (AndReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction (sve)" %}
  ins_encode %{
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           ptrue, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_andI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
  match(Set dst (AndReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
  ins_cost(2 * SVE_COST);
  format %{ "sve_reduce_andI $dst, $src1, $src2\t# andI reduction partial (sve) (may extend)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_andL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
  match(Set dst (AndReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
  ins_cost(2 * SVE_COST);
  format %{ "sve_reduce_andL $dst, $src1, $src2\t# andL reduction partial (sve)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector and reduction - predicated

instruct reduce_andI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG);
  match(Set dst (AndReductionV (Binary src1 src2) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_andI $dst, $src1, $pg, $src2\t# andI reduction predicated (sve) (may extend)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_andL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (AndReductionV (Binary src1 src2) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_andL $dst, $src1, $pg, $src2\t# andL reduction predicated (sve)" %}
  ins_encode %{
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector or reduction

instruct reduce_orI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
  match(Set dst (OrReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction (sve) (may extend)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           ptrue, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_orL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
  match(Set dst (OrReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction (sve)" %}
  ins_encode %{
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           ptrue, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_orI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
  match(Set dst (OrReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
  ins_cost(2 * SVE_COST);
  format %{ "sve_reduce_orI $dst, $src1, $src2\t# orI reduction partial (sve) (may extend)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_orL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
  match(Set dst (OrReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
  ins_cost(2 * SVE_COST);
  format %{ "sve_reduce_orL $dst, $src1, $src2\t# orL reduction partial (sve)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector or reduction - predicated

instruct reduce_orI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG);
  match(Set dst (OrReductionV (Binary src1 src2) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_orI $dst, $src1, $pg, $src2\t# orI reduction predicated (sve) (may extend)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_orL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (OrReductionV (Binary src1 src2) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_orL $dst, $src1, $pg, $src2\t# orL reduction predicated (sve)" %}
  ins_encode %{
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector xor reduction

instruct reduce_eorI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
  match(Set dst (XorReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction (sve) (may extend)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           ptrue, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_eorL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
  match(Set dst (XorReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction (sve)" %}
  ins_encode %{
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           ptrue, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_eorI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
  match(Set dst (XorReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
  ins_cost(2 * SVE_COST);
  format %{ "sve_reduce_eorI $dst, $src1, $src2\t# eorI reduction partial (sve) (may extend)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_eorL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
  match(Set dst (XorReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
  ins_cost(2 * SVE_COST);
  format %{ "sve_reduce_eorL $dst, $src1, $src2\t# eorL reduction partial (sve)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector xor reduction - predicated

instruct reduce_eorI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG);
  match(Set dst (XorReductionV (Binary src1 src2) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_eorI $dst, $src1, $pg, $src2\t# eorI reduction predicated (sve) (may extend)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_eorL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (XorReductionV (Binary src1 src2) pg));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_eorL $dst, $src1, $pg, $src2\t# eorL reduction predicated (sve)" %}
  ins_encode %{
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector max reduction

instruct reduce_maxI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
            is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
  match(Set dst (MaxReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           ptrue, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (MaxReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction (sve)" %}
  ins_encode %{
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           ptrue, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
            is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
  match(Set dst (MaxReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
  ins_cost(2 * SVE_COST);
  format %{ "sve_reduce_maxI $dst, $src1, $src2\t# maxI reduction partial (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (MaxReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
  ins_cost(2 * SVE_COST);
  format %{ "sve_reduce_maxL $dst, $src1, $src2\t# maxL reduction  partial (sve)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxF(vRegF dst, vRegF src1, vReg src2) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
  match(Set dst (MaxReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction (sve)" %}
  ins_encode %{
    __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg));
    __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxF_partial(vRegF dst, vRegF src1, vReg src2,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
  match(Set dst (MaxReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
  format %{ "sve_reduce_maxF $dst, $src1, $src2\t# maxF reduction partial (sve)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
    __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
    __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxD(vRegD dst, vRegD src1, vReg src2) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
  match(Set dst (MaxReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction (sve)" %}
  ins_encode %{
    __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
    __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxD_partial(vRegD dst, vRegD src1, vReg src2,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
  match(Set dst (MaxReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
  format %{ "sve_reduce_maxD $dst, $src1, $src2\t# maxD reduction partial (sve)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
    __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
    __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector max reduction - predicated

instruct reduce_maxI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
                           pRegGov pg, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
            is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
  match(Set dst (MaxReductionV (Binary src1 src2) pg));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_maxI $dst, $src1, $pg, $src2\t# maxI reduction predicated (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
                          pRegGov pg, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (MaxReductionV (Binary src1 src2) pg));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_maxL $dst, $src1, $pg, $src2\t# maxL reduction predicated (sve)" %}
  ins_encode %{
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (MaxReductionV (Binary src1 src2) pg));
  ins_cost(SVE_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_reduce_maxF $dst, $src1, $pg, $src2\t# maxF reduction predicated (sve)" %}
  ins_encode %{
    __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
    __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (MaxReductionV (Binary src1 src2) pg));
  ins_cost(SVE_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_reduce_maxD $dst, $src1, $pg, $src2\t# maxD reduction predicated (sve)" %}
  ins_encode %{
    __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
    __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector min reduction

instruct reduce_minI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
            is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
  match(Set dst (MinReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           ptrue, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (MinReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction (sve)" %}
  ins_encode %{
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           ptrue, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minI_partial(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD vtmp,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
            is_integral_type(n->in(2)->bottom_type()->is_vect()->element_basic_type()));
  match(Set dst (MinReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
  ins_cost(2 * SVE_COST);
  format %{ "sve_reduce_minI $dst, $src1, $src2\t# minI reduction partial (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    Assembler::SIMD_RegVariant variant = __ elemType_to_regVariant(bt);
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), variant, Matcher::vector_length(this, $src2));
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minL_partial(iRegLNoSp dst, iRegL src1, vReg src2, vRegD vtmp,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (MinReductionV src1 src2));
  effect(TEMP_DEF dst, TEMP vtmp, TEMP ptmp, KILL cr);
  ins_cost(2 * SVE_COST);
  format %{ "sve_reduce_minL $dst, $src1, $src2\t# minL reduction  partial (sve)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($ptmp$$reg), as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minF(vRegF dst, vRegF src1, vReg src2) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
  match(Set dst (MinReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction (sve)" %}
  ins_encode %{
    __ sve_fminv(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src2$$reg));
    __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minF_partial(vRegF dst, vRegF src1, vReg src2,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
  match(Set dst (MinReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
  format %{ "sve_reduce_minF $dst, $src1, $src2\t# minF reduction partial (sve)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src2));
    __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
    __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minD(vRegD dst, vRegD src1, vReg src2) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() == MaxVectorSize);
  match(Set dst (MinReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction (sve)" %}
  ins_encode %{
    __ sve_fminv(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src2$$reg));
    __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minD_partial(vRegD dst, vRegD src1, vReg src2,
                             pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() < MaxVectorSize);
  match(Set dst (MinReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst, TEMP ptmp, KILL cr);
  format %{ "sve_reduce_minD $dst, $src1, $src2\t# minD reduction partial (sve)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src2));
    __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($ptmp$$reg), as_FloatRegister($src2$$reg));
    __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector min reduction - predicated

instruct reduce_minI_masked(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp,
                           pRegGov pg, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() != T_LONG &&
            is_integral_type(n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type()));
  match(Set dst (MinReductionV (Binary src1 src2) pg));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_minI $dst, $src1, $pg, $src2\t# minI reduction predicated (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src2);
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, bt,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minL_masked(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp,
                          pRegGov pg, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (MinReductionV (Binary src1 src2) pg));
  effect(TEMP_DEF dst, TEMP tmp, KILL cr);
  ins_cost(SVE_COST);
  format %{ "sve_reduce_minL $dst, $src1, $pg, $src2\t# minL reduction predicated (sve)" %}
  ins_encode %{
    __ sve_reduce_integral(this->ideal_Opcode(), $dst$$Register, T_LONG,
                           $src1$$Register, as_FloatRegister($src2$$reg),
                           as_PRegister($pg$$reg), as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minF_masked(vRegF dst, vRegF src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (MinReductionV (Binary src1 src2) pg));
  ins_cost(SVE_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_reduce_minF $dst, $src1, $pg, $src2\t# minF reduction predicated (sve)" %}
  ins_encode %{
    __ sve_fminv(as_FloatRegister($dst$$reg), __ S, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
    __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minD_masked(vRegD dst, vRegD src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            n->in(1)->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (MinReductionV (Binary src1 src2) pg));
  ins_cost(SVE_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_reduce_minD $dst, $src1, $pg, $src2\t# minD reduction predicated (sve)" %}
  ins_encode %{
    __ sve_fminv(as_FloatRegister($dst$$reg), __ D, as_PRegister($pg$$reg), as_FloatRegister($src2$$reg));
    __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector Math.rint, floor, ceil

instruct vroundD(vReg dst, vReg src, immI rmode) %{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (RoundDoubleModeV src rmode));
  format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
  ins_encode %{
    switch ($rmode$$constant) {
      case RoundDoubleModeNode::rmode_rint:
        __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
             ptrue, as_FloatRegister($src$$reg));
        break;
      case RoundDoubleModeNode::rmode_floor:
        __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
             ptrue, as_FloatRegister($src$$reg));
        break;
      case RoundDoubleModeNode::rmode_ceil:
        __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
             ptrue, as_FloatRegister($src$$reg));
        break;
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vroundFtoI(vReg dst, vReg src, vReg tmp1, vReg tmp2, vReg tmp3, pRegGov ptmp)
%{
  predicate(UseSVE > 0);
  match(Set dst (RoundVF src));
  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP ptmp);
  format %{ "sve_vround  $dst, S, $src\t# round F to I vector" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    int vlen = Matcher::vector_length_in_bytes(this);
    if (vlen > 16) {
      __ vector_round_sve(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
                          as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
                          as_PRegister($ptmp$$reg), __ S);
    } else {
      __ vector_round_neon(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
                           as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
                           as_FloatRegister($tmp3$$reg),
                           __ esize2arrangement(type2aelembytes(bt),
                              /*isQ*/ vlen == 16));
    }
  %}
  ins_pipe(pipe_class_default);
%}

instruct vroundDtoL(vReg dst, vReg src, vReg tmp1, vReg tmp2, vReg tmp3, pRegGov ptmp)
%{
  predicate(UseSVE > 0);
  match(Set dst (RoundVD src));
  effect(TEMP_DEF dst, TEMP tmp1, TEMP tmp2, TEMP tmp3, TEMP ptmp);
  format %{ "sve_vround  $dst, D, $src\t# round D to L vector" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    int vlen = Matcher::vector_length_in_bytes(this);
    if (vlen > 16) {
      __ vector_round_sve(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
                          as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
                          as_PRegister($ptmp$$reg), __ D);
    } else {
      __ vector_round_neon(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
                           as_FloatRegister($tmp1$$reg), as_FloatRegister($tmp2$$reg),
                           as_FloatRegister($tmp3$$reg),
                           __ esize2arrangement(type2aelembytes(bt),
                              /*isQ*/ vlen == 16));
    }
  %}
  ins_pipe(pipe_class_default);
%}

// vector replicate

instruct replicateB(vReg dst, iRegIorL2I src) %{
  predicate(UseSVE > 0);
  match(Set dst (ReplicateB src));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $src\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateS(vReg dst, iRegIorL2I src) %{
  predicate(UseSVE > 0);
  match(Set dst (ReplicateS src));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $src\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateI(vReg dst, iRegIorL2I src) %{
  predicate(UseSVE > 0);
  match(Set dst (ReplicateI src));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateL(vReg dst, iRegL src) %{
  predicate(UseSVE > 0);
  match(Set dst (ReplicateL src));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateB_imm8(vReg dst, immI8 con) %{
  predicate(UseSVE > 0);
  match(Set dst (ReplicateB con));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $con\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ B, $con$$constant);
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateS_imm8(vReg dst, immI8_shift8 con) %{
  predicate(UseSVE > 0);
  match(Set dst (ReplicateS con));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $con\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ H, $con$$constant);
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateI_imm8(vReg dst, immI8_shift8 con) %{
  predicate(UseSVE > 0);
  match(Set dst (ReplicateI con));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $con\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ S, $con$$constant);
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateL_imm8(vReg dst, immL8_shift8 con) %{
  predicate(UseSVE > 0);
  match(Set dst (ReplicateL con));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $con\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ D, $con$$constant);
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateF(vReg dst, vRegF src) %{
  predicate(UseSVE > 0);
  match(Set dst (ReplicateF src));
  ins_cost(SVE_COST);
  format %{ "sve_cpy  $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateD(vReg dst, vRegD src) %{
  predicate(UseSVE > 0);
  match(Set dst (ReplicateD src));
  ins_cost(SVE_COST);
  format %{ "sve_cpy  $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector shift

instruct vasrB(vReg dst, vReg shift) %{
  predicate(UseSVE > 0);
  match(Set dst (RShiftVB dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst$$reg), __ B,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrS(vReg dst, vReg shift) %{
  predicate(UseSVE > 0);
  match(Set dst (RShiftVS dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst$$reg), __ H,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrI(vReg dst, vReg shift) %{
  predicate(UseSVE > 0);
  match(Set dst (RShiftVI dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrL(vReg dst, vReg shift) %{
  predicate(UseSVE > 0);
  match(Set dst (RShiftVL dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslB(vReg dst, vReg shift) %{
  predicate(UseSVE > 0);
  match(Set dst (LShiftVB dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslS(vReg dst, vReg shift) %{
  predicate(UseSVE > 0);
  match(Set dst (LShiftVS dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslI(vReg dst, vReg shift) %{
  predicate(UseSVE > 0);
  match(Set dst (LShiftVI dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslL(vReg dst, vReg shift) %{
  predicate(UseSVE > 0);
  match(Set dst (LShiftVL dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrB(vReg dst, vReg shift) %{
  predicate(UseSVE > 0);
  match(Set dst (URShiftVB dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrS(vReg dst, vReg shift) %{
  predicate(UseSVE > 0);
  match(Set dst (URShiftVS dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrI(vReg dst, vReg shift) %{
  predicate(UseSVE > 0);
  match(Set dst (URShiftVI dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrL(vReg dst, vReg shift) %{
  predicate(UseSVE > 0);
  match(Set dst (URShiftVL dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrB_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0);
  match(Set dst (RShiftVB src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    if (con >= 8) con = 7;
    __ sve_asr(as_FloatRegister($dst$$reg), __ B,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrS_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0);
  match(Set dst (RShiftVS src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    if (con >= 16) con = 15;
    __ sve_asr(as_FloatRegister($dst$$reg), __ H,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrI_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0);
  match(Set dst (RShiftVI src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_asr(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrL_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0);
  match(Set dst (RShiftVL src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_asr(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrB_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0);
  match(Set dst (URShiftVB src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    if (con >= 8) {
      __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrS_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0);
  match(Set dst (URShiftVS src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    if (con >= 16) {
      __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrI_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0);
  match(Set dst (URShiftVI src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrL_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0);
  match(Set dst (URShiftVL src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslB_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0);
  match(Set dst (LShiftVB src (LShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con >= 8) {
      __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslS_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0);
  match(Set dst (LShiftVS src (LShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con >= 16) {
      __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslI_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0);
  match(Set dst (LShiftVI src (LShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslL_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0);
  match(Set dst (LShiftVL src (LShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vshiftcntB(vReg dst, iRegIorL2I cnt) %{
  predicate(UseSVE > 0 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE));
  match(Set dst (LShiftCntV cnt));
  match(Set dst (RShiftCntV cnt));
  format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (B)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($cnt$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vshiftcntS(vReg dst, iRegIorL2I cnt) %{
  predicate(UseSVE > 0 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
            (n->bottom_type()->is_vect()->element_basic_type() == T_CHAR)));
  match(Set dst (LShiftCntV cnt));
  match(Set dst (RShiftCntV cnt));
  format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (H)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($cnt$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vshiftcntI(vReg dst, iRegIorL2I cnt) %{
  predicate(UseSVE > 0 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
  match(Set dst (LShiftCntV cnt));
  match(Set dst (RShiftCntV cnt));
  format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (S)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($cnt$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vshiftcntL(vReg dst, iRegIorL2I cnt) %{
  predicate(UseSVE > 0 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
  match(Set dst (LShiftCntV cnt));
  match(Set dst (RShiftCntV cnt));
  format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (D)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($cnt$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector shift - predicated

instruct vasrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (RShiftVB (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst_src1$$reg), __ B,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (RShiftVS (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst_src1$$reg), __ H,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (RShiftVI (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst_src1$$reg), __ S,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (RShiftVL (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst_src1$$reg), __ D,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (LShiftVB (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ B,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (LShiftVS (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ H,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (LShiftVI (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ S,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (LShiftVL (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst_src1$$reg), __ D,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (URShiftVB (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ B,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (URShiftVS (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ H,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (URShiftVI (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ S,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (URShiftVL (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst_src1$$reg), __ D,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (RShiftVB (Binary dst_src (RShiftCntV shift)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    assert(con > 0 && con < 8, "invalid shift immediate");
    __ sve_asr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (RShiftVS (Binary dst_src (RShiftCntV shift)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    assert(con > 0 && con < 16, "invalid shift immediate");
    __ sve_asr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (RShiftVI (Binary dst_src (RShiftCntV shift)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    assert(con > 0 && con < 32, "invalid shift immediate");
    __ sve_asr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (RShiftVL (Binary dst_src (RShiftCntV shift)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    assert(con > 0 && con < 64, "invalid shift immediate");
    __ sve_asr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (URShiftVB (Binary dst_src (RShiftCntV shift)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    assert(con > 0 && con < 8, "invalid shift immediate");
    __ sve_lsr(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (URShiftVS (Binary dst_src (RShiftCntV shift)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    assert(con > 0 && con < 16, "invalid shift immediate");
    __ sve_lsr(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (URShiftVI (Binary dst_src (RShiftCntV shift)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    assert(con > 0 && con < 32, "invalid shift immediate");
    __ sve_lsr(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (URShiftVL (Binary dst_src (RShiftCntV shift)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    assert(con > 0 && con < 64, "invalid shift immediate");
    __ sve_lsr(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslB_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (LShiftVB (Binary dst_src (LShiftCntV shift)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    assert(con >= 0 && con < 8, "invalid shift immediate");
    __ sve_lsl(as_FloatRegister($dst_src$$reg), __ B, as_PRegister($pg$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslS_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (LShiftVS (Binary dst_src (LShiftCntV shift)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    assert(con >= 0 && con < 16, "invalid shift immediate");
    __ sve_lsl(as_FloatRegister($dst_src$$reg), __ H, as_PRegister($pg$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslI_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (LShiftVI (Binary dst_src (LShiftCntV shift)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    assert(con >= 0 && con < 32, "invalid shift immediate");
    __ sve_lsl(as_FloatRegister($dst_src$$reg), __ S, as_PRegister($pg$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslL_imm_masked(vReg dst_src, immI shift, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (LShiftVL (Binary dst_src (LShiftCntV shift)) pg));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst_src, $pg, $dst_src, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    assert(con >= 0 && con < 64, "invalid shift immediate");
    __ sve_lsl(as_FloatRegister($dst_src$$reg), __ D, as_PRegister($pg$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

// vector sqrt

instruct vsqrtF(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (SqrtVF src));
  ins_cost(SVE_COST);
  format %{ "sve_fsqrt $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fsqrt(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsqrtD(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (SqrtVD src));
  ins_cost(SVE_COST);
  format %{ "sve_fsqrt $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fsqrt(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector sqrt - predicated

instruct vsqrtF_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (SqrtVF dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ S,
            as_PRegister($pg$$reg),
            as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsqrtD_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (SqrtVD dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_fsqrt $dst_src, $pg, $dst_src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fsqrt(as_FloatRegister($dst_src$$reg), __ D,
            as_PRegister($pg$$reg),
            as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector sub

instruct vsubB(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (SubVB src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst$$reg), __ B,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubS(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (SubVS src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst$$reg), __ H,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubI(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (SubVI src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubL(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (SubVL src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubF(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (SubVF src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fsub(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubD(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0);
  match(Set dst (SubVD src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fsub(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector sub - predicated

instruct vsubB_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVB (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst_src1$$reg), __ B,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubS_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVS (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst_src1$$reg), __ H,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubI_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVI (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst_src1$$reg), __ S,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubL_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVL (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst_src1$$reg), __ D,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubF_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVF (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ S,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubD_masked(vReg dst_src1, vReg src2, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src1 (SubVD (Binary dst_src1 src2) pg));
  ins_cost(SVE_COST);
  format %{ "sve_fsub $dst_src1, $pg, $dst_src1, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fsub(as_FloatRegister($dst_src1$$reg), __ D,
            as_PRegister($pg$$reg),
            as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector mask cast --------------------------

instruct vmaskcast(pRegGov dst_src) %{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
            n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
  match(Set dst_src (VectorMaskCast dst_src));
  ins_cost(0);
  format %{ "vmaskcast $dst_src\t# empty (sve)" %}
  ins_encode %{
    // empty
  %}
  ins_pipe(pipe_class_empty);
%}

instruct vmaskcast_extend(pRegGov dst, pReg src)
%{
  predicate(UseSVE > 0 &&
            (Matcher::vector_length_in_bytes(n) == 2 * Matcher::vector_length_in_bytes(n->in(1)) ||
             Matcher::vector_length_in_bytes(n) == 4 * Matcher::vector_length_in_bytes(n->in(1)) ||
             Matcher::vector_length_in_bytes(n) == 8 * Matcher::vector_length_in_bytes(n->in(1))));
  match(Set dst (VectorMaskCast src));
  ins_cost(SVE_COST * 3);
  format %{ "sve_vmaskcast_extend  $dst, $src\t# extend predicate $src" %}
  ins_encode %{
    __ sve_vmaskcast_extend(as_PRegister($dst$$reg), as_PRegister($src$$reg),
                            Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskcast_narrow(pRegGov dst, pReg src)
%{
  predicate(UseSVE > 0 &&
            (Matcher::vector_length_in_bytes(n) * 2 == Matcher::vector_length_in_bytes(n->in(1)) ||
             Matcher::vector_length_in_bytes(n) * 4 == Matcher::vector_length_in_bytes(n->in(1)) ||
             Matcher::vector_length_in_bytes(n) * 8 == Matcher::vector_length_in_bytes(n->in(1))));
  match(Set dst (VectorMaskCast src));
  ins_cost(SVE_COST * 3);
  format %{ "sve_vmaskcast_narrow  $dst, $src\t# narrow predicate $src" %}
  ins_encode %{
    __ sve_vmaskcast_narrow(as_PRegister($dst$$reg), as_PRegister($src$$reg),
                            Matcher::vector_length_in_bytes(this), Matcher::vector_length_in_bytes(this, $src));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector cast -------------------------------

instruct vcvtBtoX_extend(vReg dst, vReg src)
%{
  predicate(UseSVE > 0);
  match(Set dst (VectorCastB2X src));
  ins_cost(2 * SVE_COST);
  format %{ "sve_vectorcast_b2x  $dst, $src\t# convert B to X vector (extend)" %}
  ins_encode %{
    BasicType to_bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
    __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ B);
    if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
      __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtStoB(vReg dst, vReg src, vReg tmp)
%{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorCastS2X src));
  effect(TEMP tmp);
  ins_cost(2 * SVE_COST);
  format %{ "sve_vectorcast_s2b  $dst, $src\t# convert H to B vector" %}
  ins_encode %{
    __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
                         as_FloatRegister($src$$reg), __ H, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtStoX_extend(vReg dst, vReg src)
%{
  predicate(UseSVE > 0 &&
            type2aelembytes(Matcher::vector_element_basic_type(n)) > 2);
  match(Set dst (VectorCastS2X src));
  ins_cost(2 * SVE_COST);
  format %{ "sve_vectorcast_s2x  $dst, $src\t# convert H to X vector (extend)" %}
  ins_encode %{
    BasicType to_bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
    __ sve_vector_extend(as_FloatRegister($dst$$reg), to_size, as_FloatRegister($src$$reg), __ H);
    if (to_bt == T_FLOAT || to_bt == T_DOUBLE) {
      __ sve_scvtf(as_FloatRegister($dst$$reg), to_size, ptrue, as_FloatRegister($dst$$reg), to_size);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtItoB(vReg dst, vReg src, vReg tmp)
%{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorCastI2X src));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(3 * SVE_COST);
  format %{ "sve_vectorcast_i2b  $dst, $src\t# convert I to B vector" %}
  ins_encode %{
    __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ B,
                         as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtItoS(vReg dst, vReg src, vReg tmp)
%{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (VectorCastI2X src));
  effect(TEMP tmp);
  ins_cost(2 * SVE_COST);
  format %{ "sve_vectorcast_i2s $dst, $src\t# convert I to H vector" %}
  ins_encode %{
    __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ H,
                         as_FloatRegister($src$$reg), __ S, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtItoL(vReg dst, vReg src)
%{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (VectorCastI2X src));
  ins_cost(SVE_COST);
  format %{ "sve_vectorcast_i2l  $dst, $src\t# convert I to L vector" %}
  ins_encode %{
    __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtItoF(vReg dst, vReg src)
%{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorCastI2X src));
  ins_cost(SVE_COST);
  format %{ "sve_vectorcast_i2f  $dst, $src\t# convert I to F vector" %}
  ins_encode %{
    __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtItoD(vReg dst, vReg src)
%{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (VectorCastI2X src));
  ins_cost(2 * SVE_COST);
  format %{ "sve_vectorcast_i2d  $dst, $src\t# convert I to D vector" %}
  ins_encode %{
    __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
    __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ D);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtLtoX_narrow(vReg dst, vReg src, vReg tmp)
%{
  predicate(UseSVE > 0 && is_integral_type(Matcher::vector_element_basic_type(n)));
  match(Set dst (VectorCastL2X src));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(2 * SVE_COST);
  format %{ "sve_vectorcast_l2x  $dst, $src\t# convert L to B/H/S vector (narrow)" %}
  ins_encode %{
    BasicType to_bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
    __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
                         as_FloatRegister($src$$reg), __ D, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtLtoF(vReg dst, vReg src, vReg tmp)
%{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorCastL2X src));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(3 * SVE_COST);
  format %{ "sve_vectorcast_l2f  $dst, $src\t# convert L to F vector" %}
  ins_encode %{
    __ sve_scvtf(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
    __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
                         as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));

  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtLtoD(vReg dst, vReg src)
%{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (VectorCastL2X src));
  ins_cost(SVE_COST);
  format %{ "sve_vectorcast_l2d  $dst, $src\t# convert L to D vector" %}
  ins_encode %{
    __ sve_scvtf(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtFtoX_narrow(vReg dst, vReg src, vReg tmp)
%{
  predicate(UseSVE > 0 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT));
  match(Set dst (VectorCastF2X src));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(3 * SVE_COST);
  format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to B/H vector" %}
  ins_encode %{
    BasicType to_bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
    __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
    __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
                         as_FloatRegister($dst$$reg), __ S, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtFtoI(vReg dst, vReg src)
%{
  predicate(UseSVE > 0 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
  match(Set dst (VectorCastF2X src));
  ins_cost(SVE_COST);
  format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to I vector" %}
  ins_encode %{
    __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ S);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtFtoL(vReg dst, vReg src)
%{
  predicate(UseSVE > 0 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
  match(Set dst (VectorCastF2X src));
  ins_cost(SVE_COST * 2);
  format %{ "sve_vectorcast_f2x  $dst, $src\t# convert F to L vector" %}
  ins_encode %{
    __ sve_sunpklo(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg));
    __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtFtoD(vReg dst, vReg src)
%{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (VectorCastF2X src));
  ins_cost(2 * SVE_COST);
  format %{ "sve_vectorcast_f2d  $dst, $dst\t# convert F to D vector" %}
  ins_encode %{
    __ sve_vector_extend(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), __ S);
    __ sve_fcvt(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($dst$$reg), __ S);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtDtoX_narrow(vReg dst, vReg src, vReg tmp)
%{
  predicate(UseSVE > 0 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
             n->bottom_type()->is_vect()->element_basic_type() == T_INT));
  match(Set dst (VectorCastD2X src));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(3 * SVE_COST);
  format %{ "sve_vectorcast_d2x  $dst, $src\t# convert D to X vector (narrow)" %}
  ins_encode %{
    BasicType to_bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant to_size = __ elemType_to_regVariant(to_bt);
    __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
    __ sve_vector_narrow(as_FloatRegister($dst$$reg), to_size,
                         as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtDtoL(vReg dst, vReg src)
%{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (VectorCastD2X src));
  ins_cost(SVE_COST);
  format %{ "sve_vectorcast_d2l  $dst, $src\t# convert D to L vector" %}
  ins_encode %{
    __ sve_fcvtzs(as_FloatRegister($dst$$reg), __ D, ptrue, as_FloatRegister($src$$reg), __ D);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcvtDtoF(vReg dst, vReg src, vReg tmp)
%{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorCastD2X src));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(3 * SVE_COST);
  format %{ "sve_vectorcast_d2f  $dst, S, $dst\t# convert D to F vector" %}
  ins_encode %{
    __ sve_fcvt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg), __ D);
    __ sve_vector_narrow(as_FloatRegister($dst$$reg), __ S,
                         as_FloatRegister($dst$$reg), __ D, as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector extract ---------------------------------

instruct extractB(iRegINoSp dst, vReg src, immI idx, vReg vtmp)
%{
  predicate(UseSVE > 0 && n->in(2)->get_int() >= 16);
  match(Set dst (ExtractB src idx));
  effect(TEMP vtmp);
  ins_cost(2 * SVE_COST);
  format %{ "sve_extract_integral $dst, B, $src, $idx\t# extract from vector(B)" %}
  ins_encode %{
    __ sve_extract_integral(as_Register($dst$$reg), __ B, as_FloatRegister($src$$reg),
                            (int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct extractS(iRegINoSp dst, vReg src, immI idx, vReg vtmp)
%{
  predicate(UseSVE > 0 && n->in(2)->get_int() >= 8);
  match(Set dst (ExtractS src idx));
  effect(TEMP vtmp);
  ins_cost(2 * SVE_COST);
  format %{ "sve_extract_integral $dst, H, $src, $idx\t# extract from vector(S)" %}
  ins_encode %{
    __ sve_extract_integral(as_Register($dst$$reg), __ H, as_FloatRegister($src$$reg),
                            (int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct extractI(iRegINoSp dst, vReg src, immI idx, vReg vtmp)
%{
  predicate(UseSVE > 0 && n->in(2)->get_int() >= 4);
  match(Set dst (ExtractI src idx));
  effect(TEMP vtmp);
  ins_cost(2 * SVE_COST);
  format %{ "sve_extract_integral $dst, S, $src, $idx\t# extract from vector(I)" %}
  ins_encode %{
    __ sve_extract_integral(as_Register($dst$$reg), __ S, as_FloatRegister($src$$reg),
                            (int)($idx$$constant), /* is_signed */ true, as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct extractL(iRegLNoSp dst, vReg src, immI idx, vReg vtmp)
%{
  predicate(UseSVE > 0 && n->in(2)->get_int() >= 2);
  match(Set dst (ExtractL src idx));
  effect(TEMP vtmp);
  ins_cost(2 * SVE_COST);
  format %{ "sve_extract_integral $dst, D, $src, $idx\t# extract from vector(L)" %}
  ins_encode %{
    __ sve_extract_integral(as_Register($dst$$reg), __ D, as_FloatRegister($src$$reg),
                            (int)($idx$$constant), /* is_signed */ false, as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct extractB_LT16(iRegINoSp dst, vReg src, immI idx)
%{
  predicate(UseSVE > 0 && n->in(2)->get_int() < 16);
  match(Set dst (ExtractB src idx));
  ins_cost(INSN_COST);
  format %{ "smov $dst, B, $src, $idx\t# extract from vector(B)" %}
  ins_encode %{
    __ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ B, $idx$$constant);
  %}
  ins_pipe(pipe_class_default);
%}

instruct extractS_LT8(iRegINoSp dst, vReg src, immI idx)
%{
  predicate(UseSVE > 0 && n->in(2)->get_int() < 8);
  match(Set dst (ExtractS src idx));
  ins_cost(INSN_COST);
  format %{ "smov $dst, H, $src, $idx\t# extract from vector(S)" %}
  ins_encode %{
    __ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ H, $idx$$constant);
  %}
  ins_pipe(pipe_class_default);
%}

instruct extractI_LT4(iRegINoSp dst, vReg src, immI idx)
%{
  predicate(UseSVE > 0 && n->in(2)->get_int() < 4);
  match(Set dst (ExtractI src idx));
  ins_cost(INSN_COST);
  format %{ "smov $dst, S, $src, $idx\t# extract from vector(I)" %}
  ins_encode %{
    __ smov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ S, $idx$$constant);
  %}
  ins_pipe(pipe_class_default);
%}

instruct extractL_LT2(iRegLNoSp dst, vReg src, immI idx)
%{
  predicate(UseSVE > 0 && n->in(2)->get_int() < 2);
  match(Set dst (ExtractL src idx));
  ins_cost(INSN_COST);
  format %{ "umov $dst, D, $src, $idx\t# extract from vector(L)" %}
  ins_encode %{
    __ umov(as_Register($dst$$reg), as_FloatRegister($src$$reg), __ D, $idx$$constant);
  %}
  ins_pipe(pipe_class_default);
%}

instruct extractF(vRegF dst, vReg src, immI idx)
%{
  predicate(UseSVE > 0);
  match(Set dst (ExtractF src idx));
  ins_cost(2 * SVE_COST);
  format %{ "sve_extract_f $dst, S, $src, $idx\t# extract from vector(F)" %}
  ins_encode %{
    if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
      /* empty */
    } else if ($idx$$constant == 0) {
      __ fmovs(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
    } else if ($idx$$constant < 4) {
      __ ins(as_FloatRegister($dst$$reg), __ S, as_FloatRegister($src$$reg), 0, (int)($idx$$constant));
    } else {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
      __ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 2);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct extractD(vRegD dst, vReg src, immI idx)
%{
  predicate(UseSVE > 0);
  match(Set dst (ExtractD src idx));
  ins_cost(2 * SVE_COST);
  format %{ "sve_extract_d $dst, D, $src, $idx\t# extract from vector(D)" %}
  ins_encode %{
    if ((as_FloatRegister($dst$$reg) == as_FloatRegister($src$$reg)) && ($idx$$constant == 0)) {
      /* empty */
    } else if ($idx$$constant == 0) {
      __ fmovd(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg));
    } else if ($idx$$constant == 1) {
      __ ins(as_FloatRegister($dst$$reg), __ D, as_FloatRegister($src$$reg), 0, 1);
    } else {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
      __ sve_ext(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), $idx$$constant << 3);
    }
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------- VectorTest ----------------------------------

instruct vtest_alltrue(iRegINoSp dst, pRegGov src1, pRegGov src2, pReg ptmp, rFlagsReg cr)
%{
  predicate(UseSVE > 0 &&
            static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::overflow);
  match(Set dst (VectorTest src1 src2));
  effect(TEMP ptmp, KILL cr);
  ins_cost(SVE_COST);
  format %{ "sve_eors $ptmp, $src1, $src2\t# $src2 is all true mask\n"
            "csetw $dst, EQ\t# VectorTest (sve) - alltrue" %}
  ins_encode %{
    __ sve_eors(as_PRegister($ptmp$$reg), ptrue,
                as_PRegister($src1$$reg), as_PRegister($src2$$reg));
    __ csetw(as_Register($dst$$reg), Assembler::EQ);
  %}
  ins_pipe(pipe_slow);
%}

instruct vtest_anytrue(iRegINoSp dst, pRegGov src1, pRegGov src2, rFlagsReg cr)
%{
  predicate(UseSVE > 0 &&
            static_cast<const VectorTestNode*>(n)->get_predicate() == BoolTest::ne);
  match(Set dst (VectorTest src1 src2));
  effect(KILL cr);
  ins_cost(SVE_COST);
  format %{ "sve_ptest $src1\n\t"
            "csetw $dst, NE\t# VectorTest (sve) - anytrue" %}
  ins_encode %{
    // "src2" is not used for sve.
    __ sve_ptest(ptrue, as_PRegister($src1$$reg));
    __ csetw(as_Register($dst$$reg), Assembler::NE);
  %}
  ins_pipe(pipe_slow);
%}
// ------------------------------ Vector insert ---------------------------------

instruct insertI_le128bits(vReg dst, vReg src, iRegIorL2I val, immI idx) %{
  predicate(UseSVE > 0 &&
            (Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16) &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
             n->bottom_type()->is_vect()->element_basic_type() == T_INT));
  match(Set dst (VectorInsert (Binary src val) idx));
  ins_cost(2 * INSN_COST);
  format %{ "orr    $dst, T8/16B, $src, $src\n\t"
            "mov    $dst, B/H/S, $idx, $val\t# insertI into vector(64/128bits)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
      __ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
             as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
    }
    __ mov(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(Matcher::vector_element_basic_type(this)),
           $idx$$constant, $val$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct insertI_small_index(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
            Matcher::vector_length_in_bytes(n) > 16 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
             n->bottom_type()->is_vect()->element_basic_type() == T_INT));
  match(Set dst (VectorInsert (Binary src val) idx));
  effect(TEMP vtmp, TEMP pgtmp, KILL cr);
  ins_cost(4 * SVE_COST);
  format %{ "sve_index $vtmp, -16, 1\t# (B/H/S)\n\t"
            "sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
            "sve_orr $dst, $src, $src\n\t"
            "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ block_comment("insert into vector (B/H/S) {");
      __ sve_index(as_FloatRegister($vtmp$$reg), size, -16, 1);
      __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
                 as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
      if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
        __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
      }
      __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
    __ block_comment("} insert into vector (B/H/S)");
  %}
  ins_pipe(pipe_slow);
%}

instruct insertI(vReg dst, vReg src, iRegIorL2I val, immI idx, vReg vtmp1, vReg vtmp2, pRegGov pgtmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
            Matcher::vector_length_in_bytes(n) > 16 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE ||
             n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
             n->bottom_type()->is_vect()->element_basic_type() == T_INT));
  match(Set dst (VectorInsert (Binary src val) idx));
  effect(TEMP vtmp1, TEMP vtmp2, TEMP pgtmp, KILL cr);
  ins_cost(5 * SVE_COST);
  format %{ "sve_index $vtmp1, 0, 1\t# (B/H/S)\n\t"
            "sve_dup $vtmp2, $idx\t# (B/H/S)\n\t"
            "sve_cmpeq $pgtmp, $vtmp1, $vtmp2\n\t"
            "sve_orr $dst, $src, $src\n\t"
            "sve_cpy $dst, $pgtmp, $val\t# insert into vector (B/H/S)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ block_comment("insert into vector (B/H/S) {");
      __ sve_index(as_FloatRegister($vtmp1$$reg), size, 0, 1);
      __ sve_dup(as_FloatRegister($vtmp2$$reg), size, (int)($idx$$constant));
      __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), size, ptrue,
                 as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
      if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
        __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
      }
      __ sve_cpy(as_FloatRegister($dst$$reg), size, as_PRegister($pgtmp$$reg), $val$$Register);
    __ block_comment("} insert into vector (B/H/S)");
  %}
  ins_pipe(pipe_slow);
%}

instruct insertL_128bits(vReg dst, vReg src, iRegL val, immI idx) %{
  predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (VectorInsert (Binary src val) idx));
  ins_cost(2 * INSN_COST);
  format %{ "orr    $dst, T16B, $src, $src\n\t"
            "mov    $dst, D, $idx, $val\t# insertL into vector(128bits)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
      __ orr(as_FloatRegister($dst$$reg), __ T16B,
             as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
    }
    __ mov(as_FloatRegister($dst$$reg), __ D, $idx$$constant, $val$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct insertL(vReg dst, vReg src, iRegL val, immI idx, vReg vtmp, pRegGov pgtmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            Matcher::vector_length_in_bytes(n) > 16 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (VectorInsert (Binary src val) idx));
  effect(TEMP vtmp, TEMP pgtmp, KILL cr);
  ins_cost(4 * SVE_COST);
  format %{ "sve_index $vtmp, D, -16, 1\n\t"
            "sve_cmpeq $pgtmp, $vtmp, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
            "sve_orr $dst, $src, $src\n\t"
            "sve_cpy $dst, $pgtmp, $val\t# insert into vector (L)" %}
  ins_encode %{
    __ block_comment("insert into vector (L) {");
      __ sve_index(as_FloatRegister($vtmp$$reg), __ D, -16, 1);
      __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
                 as_FloatRegister($vtmp$$reg), (int)($idx$$constant) - 16);
      if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
        __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
      }
      __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
                 as_PRegister($pgtmp$$reg), $val$$Register);
    __ block_comment("} insert into vector (L)");
  %}
  ins_pipe(pipe_slow);
%}

instruct insertF_le128bits(vReg dst, vReg src, vRegF val, immI idx) %{
  predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
            (Matcher::vector_length_in_bytes(n) == 8 || Matcher::vector_length_in_bytes(n) == 16));
  match(Set dst (VectorInsert (Binary src val) idx));
  ins_cost(2 * INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "orr    $dst, T8/16B, $src, $src\n\t"
            "ins    $dst, S, $val, $idx, 0\t# insertF into vector(64/128bits)" %}
  ins_encode %{
    __ orr(as_FloatRegister($dst$$reg), Matcher::vector_length_in_bytes(this) == 8 ? __ T8B : __ T16B,
           as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
    __ ins(as_FloatRegister($dst$$reg), __ S,
           as_FloatRegister($val$$reg), $idx$$constant, 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct insertF_small_index(vReg dst, vReg src, vRegF val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 && n->in(2)->get_int() < 32 &&
            Matcher::vector_length_in_bytes(n) > 16 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorInsert (Binary src val) idx));
  effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
  ins_cost(4 * SVE_COST);
  format %{ "sve_index $dst, S, -16, 1\n\t"
            "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
            "sve_orr $dst, $src, $src\n\t"
            "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
  ins_encode %{
    __ block_comment("insert into vector (F) {");
      __ sve_index(as_FloatRegister($dst$$reg), __ S, -16, 1);
      __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
                 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
      __ sve_cpy(as_FloatRegister($dst$$reg), __ S, as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
    __ block_comment("} insert into vector (F)");
  %}
  ins_pipe(pipe_slow);
%}

instruct insertF(vReg dst, vReg src, vRegF val, immI idx, vReg tmp1, pRegGov pgtmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 && n->in(2)->get_int() >= 32 &&
            Matcher::vector_length_in_bytes(n) > 16 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (VectorInsert (Binary src val) idx));
  effect(TEMP_DEF dst, TEMP tmp1, TEMP pgtmp, KILL cr);
  ins_cost(5 * SVE_COST);
  format %{ "sve_index $tmp1, S, 0, 1\n\t"
            "sve_dup $dst, S, $idx\n\t"
            "sve_cmpeq $pgtmp, $tmp1, $dst\n\t"
            "sve_orr $dst, $src, $src\n\t"
            "sve_cpy $dst, $pgtmp, $val\t# insert into vector (F)" %}
  ins_encode %{
    __ block_comment("insert into vector (F) {");
      __ sve_index(as_FloatRegister($tmp1$$reg), __ S, 0, 1);
      __ sve_dup(as_FloatRegister($dst$$reg), __ S, (int)($idx$$constant));
      __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ S, ptrue,
                 as_FloatRegister($tmp1$$reg), as_FloatRegister($dst$$reg));
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
                 as_FloatRegister($src$$reg));
      __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
                 as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
    __ block_comment("} insert into vector (F)");
  %}
  ins_pipe(pipe_slow);
%}

instruct insertD_128bits(vReg dst, vReg src, vRegD val, immI idx) %{
  predicate(UseSVE > 0 && Matcher::vector_length_in_bytes(n) == 16 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (VectorInsert (Binary src val) idx));
  ins_cost(2 * INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "orr    $dst, T16B, $src, $src\n\t"
            "ins    $dst, D, $val, $idx, 0\t# insertD into vector(128bits)" %}
  ins_encode %{
    __ orr(as_FloatRegister($dst$$reg), __ T16B,
           as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
    __ ins(as_FloatRegister($dst$$reg), __ D,
           as_FloatRegister($val$$reg), $idx$$constant, 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct insertD(vReg dst, vReg src, vRegD val, immI idx, pRegGov pgtmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            Matcher::vector_length_in_bytes(n) > 16 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (VectorInsert (Binary src val) idx));
  effect(TEMP_DEF dst, TEMP pgtmp, KILL cr);
  ins_cost(4 * SVE_COST);
  format %{ "sve_index $dst, D, -16, 1\n\t"
            "sve_cmpeq $pgtmp, $dst, ($idx-#16) # shift from [0, 31] to [-16, 15]\n\t"
            "sve_orr $dst, $src, $src\n\t"
            "sve_cpy $dst, $pgtmp, $val\t# insert into vector (D)" %}
  ins_encode %{
    __ block_comment("insert into vector (D) {");
      __ sve_index(as_FloatRegister($dst$$reg), __ D, -16, 1);
      __ sve_cmp(Assembler::EQ, as_PRegister($pgtmp$$reg), __ D, ptrue,
                 as_FloatRegister($dst$$reg), (int)($idx$$constant) - 16);
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
                 as_FloatRegister($src$$reg));
      __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
                 as_PRegister($pgtmp$$reg), as_FloatRegister($val$$reg));
    __ block_comment("} insert into vector (D)");
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector shuffle -------------------------------

instruct loadshuffle(vReg dst, vReg src) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorLoadShuffle src));
  ins_cost(SVE_COST);
  format %{ "sve_loadshuffle $dst, $src\t# vector load shuffle (B/H/S/D)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (bt == T_BYTE) {
      if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
        __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
                   as_FloatRegister($src$$reg));
      }
    } else {
      __ sve_vector_extend(as_FloatRegister($dst$$reg),  __ elemType_to_regVariant(bt),
                           as_FloatRegister($src$$reg), __ B);
    }
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector rearrange -------------------------------

instruct rearrange(vReg dst, vReg src, vReg shuffle)
%{
  predicate(UseSVE > 0);
  match(Set dst (VectorRearrange src shuffle));
  ins_cost(SVE_COST);
  format %{ "sve_tbl $dst, $src, $shuffle\t# vector rearrange" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_tbl(as_FloatRegister($dst$$reg), size,
               as_FloatRegister($src$$reg), as_FloatRegister($shuffle$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector Load Gather ---------------------------------

instruct gatherI(vReg dst, indirect mem, vReg idx) %{
  predicate(UseSVE > 0 &&
            n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
  match(Set dst (LoadVectorGather mem idx));
  ins_cost(SVE_COST);
  format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (S)" %}
  ins_encode %{
    __ sve_ld1w_gather(as_FloatRegister($dst$$reg), ptrue,
                       as_Register($mem$$base), as_FloatRegister($idx$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct gatherL(vReg dst, indirect mem, vReg idx, vReg tmp) %{
  predicate(UseSVE > 0 &&
            n->as_LoadVectorGather()->memory_size() == MaxVectorSize &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
  match(Set dst (LoadVectorGather mem idx));
  effect(TEMP tmp);
  ins_cost(2 * SVE_COST);
  format %{ "load_vector_gather $dst, $mem, $idx\t# vector load gather (D)" %}
  ins_encode %{
    __ sve_uunpklo(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($idx$$reg));
    __ sve_ld1d_gather(as_FloatRegister($dst$$reg), ptrue, as_Register($mem$$base),
                       as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector Load Gather Partial-------------------------------

instruct gatherI_partial(vReg dst, indirect mem, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
  match(Set dst (LoadVectorGather mem idx));
  effect(TEMP ptmp, KILL cr);
  ins_cost(2 * SVE_COST + INSN_COST);
  format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (S)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this));
    __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
                       as_Register($mem$$base), as_FloatRegister($idx$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct gatherL_partial(vReg dst, indirect mem, vReg idx, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->as_LoadVectorGather()->memory_size() < MaxVectorSize &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
  match(Set dst (LoadVectorGather mem idx));
  effect(TEMP vtmp, TEMP ptmp, KILL cr);
  ins_cost(3 * SVE_COST + INSN_COST);
  format %{ "load_vector_gather $dst, $ptmp, $mem, $idx\t# vector load gather partial (D)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this));
    __ sve_uunpklo(as_FloatRegister($vtmp$$reg), __ D, as_FloatRegister($idx$$reg));
    __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($ptmp$$reg),
                       as_Register($mem$$base), as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector Load Gather Predicated -------------------------------

instruct gatherI_masked(vReg dst, indirect mem, vReg idx, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
  match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
  ins_cost(SVE_COST);
  format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (S)" %}
  ins_encode %{
    __ sve_ld1w_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
                       as_Register($mem$$base), as_FloatRegister($idx$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct gatherL_masked(vReg dst, indirect mem, vReg idx, pRegGov pg, vReg tmp) %{
  predicate(UseSVE > 0 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
  match(Set dst (LoadVectorGatherMasked mem (Binary idx pg)));
  effect(TEMP tmp);
  ins_cost(2 * SVE_COST);
  format %{ "load_vector_gather $dst, $pg, $mem, $idx\t# vector load gather predicated (D)" %}
  ins_encode %{
    __ sve_uunpklo(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($idx$$reg));
    __ sve_ld1d_gather(as_FloatRegister($dst$$reg), as_PRegister($pg$$reg),
                       as_Register($mem$$base), as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector Store Scatter -------------------------------

instruct scatterI(indirect mem, vReg src, vReg idx) %{
  predicate(UseSVE > 0 &&
            n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
            (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
             n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
  match(Set mem (StoreVectorScatter mem (Binary src idx)));
  ins_cost(SVE_COST);
  format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (S)" %}
  ins_encode %{
    __ sve_st1w_scatter(as_FloatRegister($src$$reg), ptrue,
                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct scatterL(indirect mem, vReg src, vReg idx, vReg tmp) %{
  predicate(UseSVE > 0 &&
            n->as_StoreVectorScatter()->memory_size() == MaxVectorSize &&
            (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
             n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
  match(Set mem (StoreVectorScatter mem (Binary src idx)));
  effect(TEMP tmp);
  ins_cost(2 * SVE_COST);
  format %{ "store_vector_scatter $mem, $idx, $src\t# vector store scatter (D)" %}
  ins_encode %{
    __ sve_uunpklo(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($idx$$reg));
    __ sve_st1d_scatter(as_FloatRegister($src$$reg), ptrue,
                        as_Register($mem$$base), as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector Store Scatter Partial -------------------------------

instruct scatterI_partial(indirect mem, vReg src, vReg idx, pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
            (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
             n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
  match(Set mem (StoreVectorScatter mem (Binary src idx)));
  effect(TEMP ptmp, KILL cr);
  ins_cost(2 * SVE_COST + INSN_COST);
  format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (S)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ S, Matcher::vector_length(this, $src));
    __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct scatterL_partial(indirect mem, vReg src, vReg idx, vReg vtmp, pRegGov ptmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->as_StoreVectorScatter()->memory_size() < MaxVectorSize &&
            (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
             n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
  match(Set mem (StoreVectorScatter mem (Binary src idx)));
  effect(TEMP vtmp, TEMP ptmp, KILL cr);
  ins_cost(3 * SVE_COST + INSN_COST);
  format %{ "store_vector_scatter $mem, $ptmp, $idx, $src\t# vector store scatter partial (D)" %}
  ins_encode %{
    __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), __ D, Matcher::vector_length(this, $src));
    __ sve_uunpklo(as_FloatRegister($vtmp$$reg), __ D, as_FloatRegister($idx$$reg));
    __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($ptmp$$reg),
                        as_Register($mem$$base), as_FloatRegister($vtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector Store Scatter Predicated -------------------------------

instruct scatterI_masked(indirect mem, vReg src, vReg idx, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_INT ||
             n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT));
  match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
  ins_cost(SVE_COST);
  format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicate (S)" %}
  ins_encode %{
    __ sve_st1w_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
                        as_Register($mem$$base), as_FloatRegister($idx$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct scatterL_masked(indirect mem, vReg src, vReg idx, pRegGov pg, vReg tmp) %{
  predicate(UseSVE > 0 &&
            (n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
             n->in(3)->in(1)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
  match(Set mem (StoreVectorScatterMasked mem (Binary src (Binary idx pg))));
  effect(TEMP tmp);
  ins_cost(2 * SVE_COST);
  format %{ "store_vector_scatter $mem, $pg, $idx, $src\t# vector store scatter predicated (D)" %}
  ins_encode %{
    __ sve_uunpklo(as_FloatRegister($tmp$$reg), __ D, as_FloatRegister($idx$$reg));
    __ sve_st1d_scatter(as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
                        as_Register($mem$$base), as_FloatRegister($tmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ Vector Load Const -------------------------------

instruct loadconB(vReg dst, immI0 src) %{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorLoadConst src));
  ins_cost(SVE_COST);
  format %{ "sve_index $dst, 0, 1\t# generate iota indices" %}
  ins_encode %{
    __ sve_index(as_FloatRegister($dst$$reg), __ B, 0, 1);
  %}
  ins_pipe(pipe_slow);
%}

// -------------------------- Populate Index to a Vector --------------------------

instruct populateindex(vReg dst, iRegIorL2I src1, immI src2) %{
  predicate(UseSVE > 0);
  match(Set dst (PopulateIndex src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_index $dst, $src1, $src2\t # populate index (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_index(as_FloatRegister($dst$$reg), __ elemType_to_regVariant(bt),
                 as_Register($src1$$reg), $src2$$constant);
  %}
  ins_pipe(pipe_slow);
%}

// ----------------------------- String Intrinsics -------------------------------

// Intrisics for String.indexOf(char)


instruct stringL_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
                                  iRegI_R0 result, vReg ztmp1, vReg ztmp2,
                                  pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
%{
  match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
  predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::L));
  effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);

  format %{ "StringLatin1 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}

  ins_encode %{
    __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
                               as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
                               as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), true /* isL */);
  %}
  ins_pipe(pipe_class_memory);
%}

instruct stringU_indexof_char_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegI_R3 ch,
                                  iRegI_R0 result, vReg ztmp1, vReg ztmp2,
                                  pRegGov pgtmp, pReg ptmp, rFlagsReg cr)
%{
  match(Set result (StrIndexOfChar (Binary str1 cnt1) ch));
  predicate((UseSVE > 0) && (((StrIndexOfCharNode*)n)->encoding() == StrIntrinsicNode::U));
  effect(TEMP ztmp1, TEMP ztmp2, TEMP pgtmp, TEMP ptmp, KILL cr);

  format %{ "StringUTF16 IndexOf char[] $str1,$cnt1,$ch -> $result # use sve" %}

  ins_encode %{
    __ string_indexof_char_sve($str1$$Register, $cnt1$$Register, $ch$$Register, $result$$Register,
                               as_FloatRegister($ztmp1$$reg), as_FloatRegister($ztmp2$$reg),
                               as_PRegister($pgtmp$$reg), as_PRegister($ptmp$$reg), false /* isL */);
  %}
  ins_pipe(pipe_class_memory);
%}

// Intrisics for String.compareTo()

// Note that Z registers alias the corresponding NEON registers, we declare the vector operands of
// these string_compare variants as NEON register type for convenience so that the prototype of
// string_compare can be shared with all variants.


instruct string_compareLL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
                              iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
                              vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
                              pRegGov_P1 pgtmp2, rFlagsReg cr)
%{
  predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LL));
  match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
  effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
         USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);

  format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
  ins_encode %{
    // Count is in 8-bit bytes; non-Compact chars are 16 bits.
    __ string_compare($str1$$Register, $str2$$Register,
                      $cnt1$$Register, $cnt2$$Register, $result$$Register,
                      $tmp1$$Register, $tmp2$$Register,
                      $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
                      as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
                      StrIntrinsicNode::LL);
  %}
  ins_pipe(pipe_class_memory);
%}

instruct string_compareLU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
                              iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
                              vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
                              pRegGov_P1 pgtmp2, rFlagsReg cr)
%{
  predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::LU));
  match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
  effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
         USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);

  format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
  ins_encode %{
    // Count is in 8-bit bytes; non-Compact chars are 16 bits.
    __ string_compare($str1$$Register, $str2$$Register,
                      $cnt1$$Register, $cnt2$$Register, $result$$Register,
                      $tmp1$$Register, $tmp2$$Register,
                      $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
                      as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
                      StrIntrinsicNode::LU);
  %}
  ins_pipe(pipe_class_memory);
%}

instruct string_compareUL_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
                              iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
                              vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
                              pRegGov_P1 pgtmp2, rFlagsReg cr)
%{
  predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UL));
  match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
  effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
         USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);

  format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
  ins_encode %{
    // Count is in 8-bit bytes; non-Compact chars are 16 bits.
    __ string_compare($str1$$Register, $str2$$Register,
                      $cnt1$$Register, $cnt2$$Register, $result$$Register,
                      $tmp1$$Register, $tmp2$$Register,
                      $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
                      as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
                      StrIntrinsicNode::UL);
  %}
  ins_pipe(pipe_class_memory);
%}

instruct string_compareUU_sve(iRegP_R1 str1, iRegI_R2 cnt1, iRegP_R3 str2, iRegI_R4 cnt2,
                              iRegI_R0 result, iRegP_R10 tmp1, iRegL_R11 tmp2,
                              vRegD_V0 vtmp1, vRegD_V1 vtmp2, pRegGov_P0 pgtmp1,
                              pRegGov_P1 pgtmp2, rFlagsReg cr)
%{
  predicate((UseSVE > 0) && (((StrCompNode*)n)->encoding() == StrIntrinsicNode::UU));
  match(Set result (StrComp (Binary str1 cnt1) (Binary str2 cnt2)));
  effect(TEMP tmp1, TEMP tmp2, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp1, TEMP pgtmp2,
         USE_KILL str1, USE_KILL str2, USE_KILL cnt1, USE_KILL cnt2, KILL cr);

  format %{ "String Compare $str1,$cnt1,$str2,$cnt2 -> $result   # USE sve" %}
  ins_encode %{
    // Count is in 8-bit bytes; non-Compact chars are 16 bits.
    __ string_compare($str1$$Register, $str2$$Register,
                      $cnt1$$Register, $cnt2$$Register, $result$$Register,
                      $tmp1$$Register, $tmp2$$Register,
                      $vtmp1$$FloatRegister, $vtmp2$$FloatRegister, fnoreg,
                      as_PRegister($pgtmp1$$reg), as_PRegister($pgtmp2$$reg),
                      StrIntrinsicNode::UU);
  %}
  ins_pipe(pipe_class_memory);
%}

// ---------------------------- Vector mask reductions ---------------------------
instruct vmask_truecount(iRegINoSp dst, pReg src) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskTrueCount src));
  ins_cost(SVE_COST);
  format %{ "vmask_truecount $dst, $src\t# vector mask truecount (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// Return the index of the first mask lane that is set, or vector length if none of
// them are set.
instruct vmask_firsttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskFirstTrue src));
  effect(TEMP ptmp);
  ins_cost(3 * SVE_COST);
  format %{ "vmask_firsttrue $dst, $src\t# vector mask firsttrue (sve)" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this, $src);
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    // When the input predicate is all-false, the result should be the vector length
    // instead of max vector register size.
    if (length_in_bytes == MaxVectorSize) {
      __ sve_brkb(as_PRegister($ptmp$$reg), ptrue, as_PRegister($src$$reg), false);
    } else {
      __ sve_ptrue_lanecnt(as_PRegister($ptmp$$reg), size, Matcher::vector_length(this, $src));
      __ sve_brkb(as_PRegister($ptmp$$reg), as_PRegister($ptmp$$reg), as_PRegister($src$$reg), false);
    }
    __ sve_cntp($dst$$Register, size, ptrue, as_PRegister($ptmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_lasttrue(iRegINoSp dst, pReg src, pReg ptmp) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskLastTrue src));
  effect(TEMP ptmp);
  ins_cost(3 * SVE_COST);
  format %{ "vmask_lasttrue $dst, $src\t# vector mask lasttrue (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this, $src);
    __ sve_vmask_lasttrue($dst$$Register, bt, as_PRegister($src$$reg), as_PRegister($ptmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_tolong(iRegLNoSp dst, pReg src, vReg vtmp1, vReg vtmp2) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskToLong src));
  effect(TEMP vtmp1, TEMP vtmp2);
  ins_cost(13 * SVE_COST);
  format %{ "vmask_tolong $dst, $src\t# vector mask tolong (sve)" %}
  ins_encode %{
    __ sve_vmask_tolong(as_Register($dst$$reg), as_PRegister($src$$reg),
                        Matcher::vector_element_basic_type(this, $src),
                        Matcher::vector_length(this, $src),
                        as_FloatRegister($vtmp1$$reg), as_FloatRegister($vtmp2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// ---------------------------- Vector mask generation ---------------------------
// The rules below set predicate registers. They can guarantee the high bits of dst
// are cleared with zero when the vector length is less than the full size of
// hardware vector register width.


// maskAll (full or partial predicate size)

instruct vmaskAll_immI(pRegGov dst, immI src) %{
  predicate(UseSVE > 0);
  match(Set dst (MaskAll src));
  ins_cost(SVE_COST);
  format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) (B/H/S)" %}
  ins_encode %{
    int con = (int)$src$$constant;
    if (con == 0) {
      __ sve_pfalse(as_PRegister($dst$$reg));
    } else {
      assert(con == -1, "invalid constant value for mask");
      BasicType bt = Matcher::vector_element_basic_type(this);
      __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
                           Matcher::vector_length(this));
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskAllI(pRegGov dst, iRegIorL2I src, vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (MaskAll src));
  effect(TEMP tmp, KILL cr);
  ins_cost(3 * SVE_COST);
  format %{ "sve_dup $tmp, $src\n\t"
            "sve_ptrue $dst, vector_length\n\t"
            "sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (B/H/S)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
    if (length_in_bytes < MaxVectorSize) {
      __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
      __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
                 as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
    } else {
      __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskAll_immL(pRegGov dst, immL src) %{
  predicate(UseSVE > 0);
  match(Set dst (MaskAll src));
  ins_cost(SVE_COST);
  format %{ "sve_ptrue_lanecnt/sve_pfalse $dst\t# mask all (sve) (D)" %}
  ins_encode %{
    long con = (long)$src$$constant;
    if (con == 0) {
      __ sve_pfalse(as_PRegister($dst$$reg));
    } else {
      assert(con == -1, "invalid constant value for mask");
      BasicType bt = Matcher::vector_element_basic_type(this);
      __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
                           Matcher::vector_length(this));
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskAllL(pRegGov dst, iRegL src, vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (MaskAll src));
  effect(TEMP tmp, KILL cr);
  ins_cost(3 * SVE_COST);
  format %{ "sve_dup $tmp, $src\n\t"
            "sve_ptrue $dst, vector_length\n\t"
            "sve_cmpne $dst, $dst, $tmp, 0\t# mask all (sve) (D)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    __ sve_dup(as_FloatRegister($tmp$$reg), size, as_Register($src$$reg));
    if (length_in_bytes < MaxVectorSize) {
      __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), size, Matcher::vector_length(this));
      __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size,
                 as_PRegister($dst$$reg), as_FloatRegister($tmp$$reg), 0);
    } else {
      __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector mask compare

instruct vmaskcmp(pRegGov dst, vReg src1, vReg src2, immI cond, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskCmp (Binary src1 src2) cond));
  effect(KILL cr);
  ins_cost(2 * SVE_COST);
  format %{ "sve_cmp $dst, $src1, $src2\t# vector mask cmp (sve)" %}
  ins_encode %{
    uint length_in_bytes = Matcher::vector_length_in_bytes(this);
    BasicType bt = Matcher::vector_element_basic_type(this);
    if (length_in_bytes == MaxVectorSize) {
      __ sve_compare(as_PRegister($dst$$reg), bt, ptrue, as_FloatRegister($src1$$reg),
                     as_FloatRegister($src2$$reg), (int)$cond$$constant);
    } else {
      __ sve_ptrue_lanecnt(as_PRegister($dst$$reg), __ elemType_to_regVariant(bt),
                           Matcher::vector_length(this));
      __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($dst$$reg), as_FloatRegister($src1$$reg),
                     as_FloatRegister($src2$$reg), (int)$cond$$constant);
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmaskcmp_masked(pRegGov dst, vReg src1, vReg src2, immI cond, pRegGov pg, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (VectorMaskCmp (Binary src1 src2) (Binary cond pg)));
  effect(KILL cr);
  ins_cost(SVE_COST);
  format %{ "sve_cmp $dst, $pg, $src1, $src2\t# vector mask cmp (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    __ sve_compare(as_PRegister($dst$$reg), bt, as_PRegister($pg$$reg), as_FloatRegister($src1$$reg),
                   as_FloatRegister($src2$$reg), (int)$cond$$constant);
  %}
  ins_pipe(pipe_slow);
%}

// vector load mask

instruct vloadmaskB(pRegGov dst, vReg src, rFlagsReg cr) %{
  predicate(UseSVE > 0 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (VectorLoadMask src));
  effect(KILL cr);
  ins_cost(SVE_COST);
  format %{ "vloadmaskB $dst, $src\t# vector load mask (sve) (B)" %}
  ins_encode %{
    __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), __ B,
               ptrue, as_FloatRegister($src$$reg), 0);
  %}
  ins_pipe(pipe_slow);
%}

instruct vloadmask_extend(pRegGov dst, vReg src, vReg tmp, rFlagsReg cr) %{
  predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() != T_BYTE);
  match(Set dst (VectorLoadMask src));
  effect(TEMP tmp, KILL cr);
  ins_cost(3 * SVE_COST);
  format %{ "vloadmask $dst, $src\t# vector load mask (sve) (H/S/D)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_vector_extend(as_FloatRegister($tmp$$reg), size, as_FloatRegister($src$$reg), __ B);
    __ sve_cmp(Assembler::NE, as_PRegister($dst$$reg), size, ptrue, as_FloatRegister($tmp$$reg), 0);
  %}
  ins_pipe(pipe_slow);
%}

// ---------------------------- Compress/Expand Operations ---------------------------

instruct mcompress(pReg dst, pReg pg, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set dst (CompressM pg));
  effect(KILL cr);
  ins_cost(2 * SVE_COST);
  format %{ "sve_cntp rscratch1, $pg\n\t"
            "sve_whilelo $dst, zr, rscratch1\t# mask compress (B/H/S/D)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_cntp(rscratch1, size, ptrue, as_PRegister($pg$$reg));
    __ sve_whilelo(as_PRegister($dst$$reg), size, zr, rscratch1);
  %}
  ins_pipe(pipe_slow);
%}

instruct vcompress(vReg dst, vReg src, pRegGov pg) %{
  predicate(UseSVE > 0 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_INT ||
             n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT ||
             n->bottom_type()->is_vect()->element_basic_type() == T_LONG ||
             n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE));
  match(Set dst (CompressV src pg));
  ins_cost(SVE_COST);
  format %{ "sve_compact $dst, $src, $pg\t# vector compress (S/D)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_compact(as_FloatRegister($dst$$reg), size, as_FloatRegister($src$$reg), as_PRegister($pg$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcompressB(vReg dst, vReg src, pReg pg, vReg vtmp1, vReg vtmp2, vReg vtmp3, vReg vtmp4,
                    pReg ptmp, pRegGov pgtmp) %{
  predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  effect(TEMP_DEF dst, TEMP vtmp1, TEMP vtmp2, TEMP vtmp3, TEMP vtmp4, TEMP ptmp, TEMP pgtmp);
  match(Set dst (CompressV src pg));
  ins_cost(13 * SVE_COST);
  format %{ "sve_compact $dst, $src, $pg\t# vector compress (B)" %}
  ins_encode %{
    __ sve_compress_byte(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
                         as_FloatRegister($vtmp1$$reg),as_FloatRegister($vtmp2$$reg),
                         as_FloatRegister($vtmp3$$reg),as_FloatRegister($vtmp4$$reg),
                         as_PRegister($ptmp$$reg), as_PRegister($pgtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vcompressS(vReg dst, vReg src, pReg pg, vReg vtmp1, vReg vtmp2, pRegGov pgtmp) %{
  predicate(UseSVE > 0 && n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  effect(TEMP_DEF dst, TEMP vtmp1, TEMP vtmp2, TEMP pgtmp);
  match(Set dst (CompressV src pg));
  ins_cost(38 * SVE_COST);
  format %{ "sve_compact $dst, $src, $pg\t# vector compress (H)" %}
  ins_encode %{
    __ sve_compress_short(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_PRegister($pg$$reg),
                          as_FloatRegister($vtmp1$$reg),as_FloatRegister($vtmp2$$reg), as_PRegister($pgtmp$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vexpand(vReg dst, vReg src, pRegGov pg) %{
  match(Set dst (ExpandV src pg));
  effect(TEMP_DEF dst);
  ins_cost(4 * SVE_COST);
  format %{ "sve_dup $dst, S/D, 0\n\t"
            "sve_histcnt $dst, S/D, $pg, $dst, $dst\n\t"
            "sve_sub $dst, S/D, 1\n\t"
            "sve_tbl $dst, S/D, $src, $dst\t# vector expand (S/D)" %}
  ins_encode %{
    // Example input:   src   = 1 2 3 4 5 6 7 8
    //                  pg    = 1 0 0 1 1 0 1 1
    // Expected result: dst   = 4 0 0 5 6 0 7 8

    // The basic idea is to use TBL which can shuffle the elements in the given
    // vector flexibly. HISTCNT + SUB is used to generate the second source input
    // for TBL whose value is used to select the indexed element from src vector.

    BasicType bt = Matcher::vector_element_basic_type(this);
    assert(UseSVE == 2 && !is_subword_type(bt), "unsupported");
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    // dst = 0 0 0 0 0 0 0 0
    __ sve_dup(as_FloatRegister($dst$$reg), size, 0);
    // dst = 5 0 0 4 3 0 2 1
    __ sve_histcnt(as_FloatRegister($dst$$reg), size, as_PRegister($pg$$reg),
                   as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg));
    // dst = 4 -1 -1 3 2 -1 1 0
    __ sve_sub(as_FloatRegister($dst$$reg), size, 1);
    // dst = 4 0 0 5 6 0 7 8
    __ sve_tbl(as_FloatRegister($dst$$reg), size, as_FloatRegister($src$$reg),
               as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmask_gen(pRegGov pg, iRegL len, rFlagsReg cr) %{
  predicate(UseSVE > 0);
  match(Set pg (VectorMaskGen len));
  effect(KILL cr);
  ins_cost(SVE_COST);
  format %{ "sve_whilelo $pg, zr, $len\t # sve" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_whilelo(as_PRegister($pg$$reg), size, zr, as_Register($len$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ CountLeadingZerosV ------------------------------

instruct vcountLeadingZeros(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (CountLeadingZerosV src));
  ins_cost(SVE_COST);
  format %{ "sve_clz $dst, $src\t# vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_clz(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// The dst and src should use the same register to make sure the
// inactive lanes in dst save the same elements as src.
instruct vcountLeadingZeros_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (CountLeadingZerosV dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_clz $dst_src, $pg, $dst_src\t# vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_clz(as_FloatRegister($dst_src$$reg), size,
        as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// ------------------------------ CountTrailingZerosV -----------------------------

instruct vcountTrailingZeros(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (CountTrailingZerosV src));
  ins_cost(2 * SVE_COST);
  format %{ "sve_rbit $dst, $src\n\t"
            "sve_clz  $dst, $dst\t# vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_rbit(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg));
    __ sve_clz(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($dst$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// The dst and src should use the same register to make sure the
// inactive lanes in dst save the same elements as src.
instruct vcountTrailingZeros_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (CountTrailingZerosV dst_src pg));
  ins_cost(2 * SVE_COST);
  format %{ "sve_rbit $dst_src, $pg, $dst_src\n\t"
            "sve_clz  $dst_src, $pg, $dst_src\t# vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_rbit(as_FloatRegister($dst_src$$reg), size,
        as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
    __ sve_clz(as_FloatRegister($dst_src$$reg), size,
        as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// ---------------------------------- ReverseV ------------------------------------

instruct vreverse(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (ReverseV src));
  ins_cost(SVE_COST);
  format %{ "sve_rbit $dst, $src\t# vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_rbit(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// The dst and src should use the same register to make sure the
// inactive lanes in dst save the same elements as src.
instruct vreverse_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (ReverseV dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_rbit $dst_src, $pg, $dst_src\t# vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    __ sve_rbit(as_FloatRegister($dst_src$$reg), size,
        as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// -------------------------------- ReverseBytesV ---------------------------------

instruct vreverseBytes(vReg dst, vReg src) %{
  predicate(UseSVE > 0 &&
            !n->as_Vector()->is_predicated_vector());
  match(Set dst (ReverseBytesV src));
  ins_cost(SVE_COST);
  format %{ "sve_revb $dst, $src\t# vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    if (bt == T_BYTE) {
      if (as_FloatRegister($dst$$reg) != as_FloatRegister($src$$reg)) {
        __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg), as_FloatRegister($src$$reg));
      }
    } else {
      __ sve_revb(as_FloatRegister($dst$$reg), size, ptrue, as_FloatRegister($src$$reg));
    }
  %}
  ins_pipe(pipe_slow);
%}

// The dst and src should use the same register to make sure the
// inactive lanes in dst save the same elements as src.
instruct vreverseBytes_masked(vReg dst_src, pRegGov pg) %{
  predicate(UseSVE > 0);
  match(Set dst_src (ReverseBytesV dst_src pg));
  ins_cost(SVE_COST);
  format %{ "sve_revb $dst_src, $pg, $dst_src\t# vector (sve)" %}
  ins_encode %{
    BasicType bt = Matcher::vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = __ elemType_to_regVariant(bt);
    if (bt == T_BYTE) {
      // do nothing
    } else {
      __ sve_revb(as_FloatRegister($dst_src$$reg), size,
          as_PRegister($pg$$reg), as_FloatRegister($dst_src$$reg));
    }
  %}
  ins_pipe(pipe_slow);
%}