//
// Copyright (c) 2020, 2021, Oracle and/or its affiliates. All rights reserved.
// Copyright (c) 2020, 2021, Arm Limited. All rights reserved.
// DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
//
// This code is free software; you can redistribute it and/or modify it
// under the terms of the GNU General Public License version 2 only, as
// published by the Free Software Foundation.
//
// This code is distributed in the hope that it will be useful, but WITHOUT
// ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
// FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
// version 2 for more details (a copy is included in the LICENSE file that
// accompanied this code).
//
// You should have received a copy of the GNU General Public License version
// 2 along with this work; if not, write to the Free Software Foundation,
// Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
//
// Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
// or visit www.oracle.com if you need additional information or have any
// questions.
//
//

// This file is automatically generated by running "m4 aarch64_sve_ad.m4". Do not edit ----

// AArch64 SVE Architecture Description File


// 4 bit signed offset -- for predicated load/store

operand vmemA_immIOffset4()
%{
  predicate(Address::offset_ok_for_sve_immed(n->get_int(), 4,
            Matcher::scalable_vector_reg_size(T_BYTE)));
  match(ConI);

  op_cost(0);
  format %{ %}
  interface(CONST_INTER);
%}

operand vmemA_immLOffset4()
%{
  predicate(Address::offset_ok_for_sve_immed(n->get_long(), 4,
            Matcher::scalable_vector_reg_size(T_BYTE)));
  match(ConL);

  op_cost(0);
  format %{ %}
  interface(CONST_INTER);
%}

operand vmemA_indOffI4(iRegP reg, vmemA_immIOffset4 off)
%{
  constraint(ALLOC_IN_RC(ptr_reg));
  match(AddP reg off);
  op_cost(0);
  format %{ "[$reg, $off, MUL VL]" %}
  interface(MEMORY_INTER) %{
    base($reg);
    index(0xffffffff);
    scale(0x0);
    disp($off);
  %}
%}

operand vmemA_indOffL4(iRegP reg, vmemA_immLOffset4 off)
%{
  constraint(ALLOC_IN_RC(ptr_reg));
  match(AddP reg off);
  op_cost(0);
  format %{ "[$reg, $off, MUL VL]" %}
  interface(MEMORY_INTER) %{
    base($reg);
    index(0xffffffff);
    scale(0x0);
    disp($off);
  %}
%}

opclass vmemA(indirect, vmemA_indOffI4, vmemA_indOffL4);

source_hpp %{
  bool op_sve_supported(int opcode);
%}

source %{
  static inline BasicType vector_element_basic_type(const MachNode* n) {
    const TypeVect* vt = n->bottom_type()->is_vect();
    return vt->element_basic_type();
  }

  static inline BasicType vector_element_basic_type(const MachNode* use, const MachOper* opnd) {
    int def_idx = use->operand_index(opnd);
    Node* def = use->in(def_idx);
    const TypeVect* vt = def->bottom_type()->is_vect();
    return vt->element_basic_type();
  }

  static Assembler::SIMD_RegVariant elemBytes_to_regVariant(int esize) {
    switch(esize) {
      case 1:
        return Assembler::B;
      case 2:
        return Assembler::H;
      case 4:
        return Assembler::S;
      case 8:
        return Assembler::D;
      default:
        assert(false, "unsupported");
        ShouldNotReachHere();
    }
    return Assembler::INVALID;
  }

  static Assembler::SIMD_RegVariant elemType_to_regVariant(BasicType bt) {
    return elemBytes_to_regVariant(type2aelembytes(bt));
  }

  typedef void (C2_MacroAssembler::* sve_mem_insn_predicate)(FloatRegister Rt, Assembler::SIMD_RegVariant T,
                                                             PRegister Pg, const Address &adr);

  // Predicated load/store, with optional ptrue to all elements of given predicate register.
  static void loadStoreA_predicate(C2_MacroAssembler masm, bool is_store,
                                   FloatRegister reg, PRegister pg, BasicType bt,
                                   int opcode, Register base, int index, int size, int disp) {
    sve_mem_insn_predicate insn;
    Assembler::SIMD_RegVariant type;
    int esize = type2aelembytes(bt);
    if (index == -1) {
      assert(size == 0, "unsupported address mode: scale size = %d", size);
      switch(esize) {
      case 1:
        insn = is_store ? &C2_MacroAssembler::sve_st1b : &C2_MacroAssembler::sve_ld1b;
        type = Assembler::B;
        break;
      case 2:
        insn = is_store ? &C2_MacroAssembler::sve_st1h : &C2_MacroAssembler::sve_ld1h;
        type = Assembler::H;
        break;
      case 4:
        insn = is_store ? &C2_MacroAssembler::sve_st1w : &C2_MacroAssembler::sve_ld1w;
        type = Assembler::S;
        break;
      case 8:
        insn = is_store ? &C2_MacroAssembler::sve_st1d : &C2_MacroAssembler::sve_ld1d;
        type = Assembler::D;
        break;
      default:
        assert(false, "unsupported");
        ShouldNotReachHere();
      }
      (masm.*insn)(reg, type, pg, Address(base, disp / Matcher::scalable_vector_reg_size(T_BYTE)));
    } else {
      assert(false, "unimplemented");
      ShouldNotReachHere();
    }
  }

  bool op_sve_supported(int opcode) {
    switch (opcode) {
      case Op_MulAddVS2VI:
        // No multiply reduction instructions
      case Op_MulReductionVD:
      case Op_MulReductionVF:
      case Op_MulReductionVI:
      case Op_MulReductionVL:
        // Others
      case Op_Extract:
      case Op_ExtractB:
      case Op_ExtractC:
      case Op_ExtractD:
      case Op_ExtractF:
      case Op_ExtractI:
      case Op_ExtractL:
      case Op_ExtractS:
      case Op_ExtractUB:
      // Vector API specific
      case Op_AndReductionV:
      case Op_OrReductionV:
      case Op_XorReductionV:
      case Op_MaxReductionV:
      case Op_MinReductionV:
      case Op_LoadVectorGather:
      case Op_StoreVectorScatter:
      case Op_VectorBlend:
      case Op_VectorCast:
      case Op_VectorCastB2X:
      case Op_VectorCastD2X:
      case Op_VectorCastF2X:
      case Op_VectorCastI2X:
      case Op_VectorCastL2X:
      case Op_VectorCastS2X:
      case Op_VectorInsert:
      case Op_VectorLoadConst:
      case Op_VectorLoadMask:
      case Op_VectorLoadShuffle:
      case Op_VectorMaskCmp:
      case Op_VectorRearrange:
      case Op_VectorReinterpret:
      case Op_VectorStoreMask:
      case Op_VectorTest:
        return false;
      default:
        return true;
    }
  }
%}

definitions %{
  int_def SVE_COST             (200, 200);
%}


// All SVE instructions

// vector load/store

// Use predicated vector load/store
instruct loadV(vReg dst, vmemA mem) %{
  predicate(UseSVE > 0 && n->as_LoadVector()->memory_size() >= 16);
  match(Set dst (LoadVector mem));
  ins_cost(SVE_COST);
  format %{ "sve_ldr $dst, $mem\t # vector (sve)" %}
  ins_encode %{
    FloatRegister dst_reg = as_FloatRegister($dst$$reg);
    loadStoreA_predicate(C2_MacroAssembler(&cbuf), false, dst_reg, ptrue,
                         vector_element_basic_type(this), $mem->opcode(),
                         as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

instruct storeV(vReg src, vmemA mem) %{
  predicate(UseSVE > 0 && n->as_StoreVector()->memory_size() >= 16);
  match(Set mem (StoreVector mem src));
  ins_cost(SVE_COST);
  format %{ "sve_str $mem, $src\t # vector (sve)" %}
  ins_encode %{
    FloatRegister src_reg = as_FloatRegister($src$$reg);
    loadStoreA_predicate(C2_MacroAssembler(&cbuf), true, src_reg, ptrue,
                         vector_element_basic_type(this, $src), $mem->opcode(),
                         as_Register($mem$$base), $mem$$index, $mem$$scale, $mem$$disp);
  %}
  ins_pipe(pipe_slow);
%}

// vector abs

instruct vabsB(vReg dst, vReg src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (AbsVB src));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst, $src\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst$$reg), __ B,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsS(vReg dst, vReg src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (AbsVS src));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst, $src\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst$$reg), __ H,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsI(vReg dst, vReg src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (AbsVI src));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsL(vReg dst, vReg src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (AbsVL src));
  ins_cost(SVE_COST);
  format %{ "sve_abs $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_abs(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsF(vReg dst, vReg src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_FLOAT);
  match(Set dst (AbsVF src));
  ins_cost(SVE_COST);
  format %{ "sve_fabs $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fabs(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vabsD(vReg dst, vReg src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (AbsVD src));
  ins_cost(SVE_COST);
  format %{ "sve_fabs $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fabs(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector add

instruct vaddB(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (AddVB src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst$$reg), __ B,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddS(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (AddVS src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst$$reg), __ H,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddI(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (AddVI src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddL(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (AddVL src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_add $dst, $src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_add(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddF(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (AddVF src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fadd(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vaddD(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (AddVD src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fadd $dst, $src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fadd(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector and

instruct vand(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (AndV src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_and  $dst, $src1, $src2\t# vector (sve)" %}
  ins_encode %{
    __ sve_and(as_FloatRegister($dst$$reg),
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector or

instruct vor(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (OrV src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_orr  $dst, $src1, $src2\t# vector (sve)" %}
  ins_encode %{
    __ sve_orr(as_FloatRegister($dst$$reg),
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector xor

instruct vxor(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (XorV src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_eor  $dst, $src1, $src2\t# vector (sve)" %}
  ins_encode %{
    __ sve_eor(as_FloatRegister($dst$$reg),
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector not

instruct vnotI(vReg dst, vReg src, immI_M1 m1) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (XorV src (ReplicateB m1)));
  match(Set dst (XorV src (ReplicateS m1)));
  match(Set dst (XorV src (ReplicateI m1)));
  ins_cost(SVE_COST);
  format %{ "sve_not $dst, $src\t# vector (sve) B/H/S" %}
  ins_encode %{
    __ sve_not(as_FloatRegister($dst$$reg), __ D,
               ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vnotL(vReg dst, vReg src, immL_M1 m1) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (XorV src (ReplicateL m1)));
  ins_cost(SVE_COST);
  format %{ "sve_not $dst, $src\t# vector (sve) D" %}
  ins_encode %{
    __ sve_not(as_FloatRegister($dst$$reg), __ D,
               ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}


// vector and_not

instruct vand_notI(vReg dst, vReg src1, vReg src2, immI_M1 m1) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (AndV src1 (XorV src2 (ReplicateB m1))));
  match(Set dst (AndV src1 (XorV src2 (ReplicateS m1))));
  match(Set dst (AndV src1 (XorV src2 (ReplicateI m1))));
  ins_cost(SVE_COST);
  format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) B/H/S" %}
  ins_encode %{
    __ sve_bic(as_FloatRegister($dst$$reg),
               as_FloatRegister($src1$$reg),
               as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vand_notL(vReg dst, vReg src1, vReg src2, immL_M1 m1) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (AndV src1 (XorV src2 (ReplicateL m1))));
  ins_cost(SVE_COST);
  format %{ "sve_bic $dst, $src1, $src2\t# vector (sve) D" %}
  ins_encode %{
    __ sve_bic(as_FloatRegister($dst$$reg),
               as_FloatRegister($src1$$reg),
               as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}


// vector float div

instruct vdivF(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (DivVF dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vdivD(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (DivVD dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fdiv  $dst_src1, $dst_src1, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fdiv(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector min/max

instruct vmin(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst_src1 (MinV dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_min $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
  ins_encode %{
    BasicType bt = vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
    if (is_floating_point_type(bt)) {
      __ sve_fmin(as_FloatRegister($dst_src1$$reg), size,
                  ptrue, as_FloatRegister($src2$$reg));
    } else {
      assert(is_integral_type(bt), "Unsupported type");
      __ sve_smin(as_FloatRegister($dst_src1$$reg), size,
                  ptrue, as_FloatRegister($src2$$reg));
    }
  %}
  ins_pipe(pipe_slow);
%}

instruct vmax(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst_src1 (MaxV dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_max $dst_src1, $dst_src1, $src2\t # vector (sve)" %}
  ins_encode %{
    BasicType bt = vector_element_basic_type(this);
    Assembler::SIMD_RegVariant size = elemType_to_regVariant(bt);
    if (is_floating_point_type(bt)) {
      __ sve_fmax(as_FloatRegister($dst_src1$$reg), size,
                  ptrue, as_FloatRegister($src2$$reg));
    } else {
      assert(is_integral_type(bt), "Unsupported type");
      __ sve_smax(as_FloatRegister($dst_src1$$reg), size,
                  ptrue, as_FloatRegister($src2$$reg));
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector fmla

// dst_src1 = dst_src1 + src2 * src3
instruct vfmlaF(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (FmaVF dst_src1 (Binary src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * src3
instruct vfmlaD(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (FmaVD dst_src1 (Binary src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fmla(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector fmls

// dst_src1 = dst_src1 + -src2 * src3
// dst_src1 = dst_src1 + src2 * -src3
instruct vfmlsF(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (FmaVF dst_src1 (Binary (NegVF src2) src3)));
  match(Set dst_src1 (FmaVF dst_src1 (Binary src2 (NegVF src3))));
  ins_cost(SVE_COST);
  format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + -src2 * src3
// dst_src1 = dst_src1 + src2 * -src3
instruct vfmlsD(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (FmaVD dst_src1 (Binary (NegVD src2) src3)));
  match(Set dst_src1 (FmaVD dst_src1 (Binary src2 (NegVD src3))));
  ins_cost(SVE_COST);
  format %{ "sve_fmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fmls(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector fnmla

// dst_src1 = -dst_src1 + -src2 * src3
// dst_src1 = -dst_src1 + src2 * -src3
instruct vfnmlaF(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary (NegVF src2) src3)));
  match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 (NegVF src3))));
  ins_cost(SVE_COST);
  format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = -dst_src1 + -src2 * src3
// dst_src1 = -dst_src1 + src2 * -src3
instruct vfnmlaD(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary (NegVD src2) src3)));
  match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 (NegVD src3))));
  ins_cost(SVE_COST);
  format %{ "sve_fnmla $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fnmla(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector fnmls

// dst_src1 = -dst_src1 + src2 * src3
instruct vfnmlsF(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (FmaVF (NegVF dst_src1) (Binary src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = -dst_src1 + src2 * src3
instruct vfnmlsD(vReg dst_src1, vReg src2, vReg src3) %{
  predicate(UseFMA && UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (FmaVD (NegVD dst_src1) (Binary src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_fnmls $dst_src1, $src2, $src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fnmls(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector mla

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaB(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst_src1 (AddVB dst_src1 (MulVB src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ B,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaS(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst_src1 (AddVS dst_src1 (MulVS src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ H,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaI(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (AddVI dst_src1 (MulVI src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ S,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 + src2 * src3
instruct vmlaL(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (AddVL dst_src1 (MulVL src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mla $dst_src1, src2, src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_mla(as_FloatRegister($dst_src1$$reg), __ D,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector mls

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsB(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst_src1 (SubVB dst_src1 (MulVB src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ B,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsS(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst_src1 (SubVS dst_src1 (MulVS src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ H,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsI(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (SubVI dst_src1 (MulVI src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ S,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// dst_src1 = dst_src1 - src2 * src3
instruct vmlsL(vReg dst_src1, vReg src2, vReg src3)
%{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (SubVL dst_src1 (MulVL src2 src3)));
  ins_cost(SVE_COST);
  format %{ "sve_mls $dst_src1, src2, src3\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_mls(as_FloatRegister($dst_src1$$reg), __ D,
      ptrue, as_FloatRegister($src2$$reg), as_FloatRegister($src3$$reg));
  %}
  ins_pipe(pipe_slow);
%}


// vector mul

instruct vmulB(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst_src1 (MulVB dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ B,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulS(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst_src1 (MulVS dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ H,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulI(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst_src1 (MulVI dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulL(vReg dst_src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst_src1 (MulVL dst_src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_mul $dst_src1, $dst_src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_mul(as_FloatRegister($dst_src1$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulF(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (MulVF src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fmul(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vmulD(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (MulVD src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fmul $dst, $src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fmul(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector fneg

instruct vnegF(vReg dst, vReg src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (NegVF src));
  ins_cost(SVE_COST);
  format %{ "sve_fneg $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fneg(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vnegD(vReg dst, vReg src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (NegVD src));
  ins_cost(SVE_COST);
  format %{ "sve_fneg $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fneg(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// popcount vector

instruct vpopcountI(vReg dst, vReg src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (PopCountVI src));
  format %{ "sve_cnt $dst, $src\t# vector (sve) (S)\n\t"  %}
  ins_encode %{
     __ sve_cnt(as_FloatRegister($dst$$reg), __ S, ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector add reduction

instruct reduce_addB(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_BYTE);
  match(Set dst (AddReductionVI src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_uaddv $tmp, $src2\t# vector (sve) (B)\n\t"
            "smov  $dst, $tmp, B, 0\n\t"
            "addw  $dst, $dst, $src1\n\t"
            "sxtb  $dst, $dst\t # add reduction B" %}
  ins_encode %{
    __ sve_uaddv(as_FloatRegister($tmp$$reg), __ B,
         ptrue, as_FloatRegister($src2$$reg));
    __ smov($dst$$Register, as_FloatRegister($tmp$$reg), __ B, 0);
    __ addw($dst$$Register, $dst$$Register, $src1$$Register);
    __ sxtb($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addS(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_SHORT);
  match(Set dst (AddReductionVI src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_uaddv $tmp, $src2\t# vector (sve) (H)\n\t"
            "smov  $dst, $tmp, H, 0\n\t"
            "addw  $dst, $dst, $src1\n\t"
            "sxth  $dst, $dst\t # add reduction H" %}
  ins_encode %{
    __ sve_uaddv(as_FloatRegister($tmp$$reg), __ H,
         ptrue, as_FloatRegister($src2$$reg));
    __ smov($dst$$Register, as_FloatRegister($tmp$$reg), __ H, 0);
    __ addw($dst$$Register, $dst$$Register, $src1$$Register);
    __ sxth($dst$$Register, $dst$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addI(iRegINoSp dst, iRegIorL2I src1, vReg src2, vRegD tmp) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_INT);
  match(Set dst (AddReductionVI src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_uaddv $tmp, $src2\t# vector (sve) (S)\n\t"
            "umov  $dst, $tmp, S, 0\n\t"
            "addw  $dst, $dst, $src1\t # add reduction S" %}
  ins_encode %{
    __ sve_uaddv(as_FloatRegister($tmp$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
    __ umov($dst$$Register, as_FloatRegister($tmp$$reg), __ S, 0);
    __ addw($dst$$Register, $dst$$Register, $src1$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addL(iRegLNoSp dst, iRegL src1, vReg src2, vRegD tmp) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16 &&
            n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_LONG);
  match(Set dst (AddReductionVL src1 src2));
  effect(TEMP_DEF dst, TEMP tmp);
  ins_cost(SVE_COST);
  format %{ "sve_uaddv $tmp, $src2\t# vector (sve) (D)\n\t"
            "umov  $dst, $tmp, D, 0\n\t"
            "add  $dst, $dst, $src1\t # add reduction D" %}
  ins_encode %{
    __ sve_uaddv(as_FloatRegister($tmp$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
    __ umov($dst$$Register, as_FloatRegister($tmp$$reg), __ D, 0);
    __ add($dst$$Register, $dst$$Register, $src1$$Register);
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addF(vRegF src1_dst, vReg src2) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16);
  match(Set src1_dst (AddReductionVF src1_dst src2));
  ins_cost(SVE_COST);
  format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_addD(vRegD src1_dst, vReg src2) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16);
  match(Set src1_dst (AddReductionVD src1_dst src2));
  ins_cost(SVE_COST);
  format %{ "sve_fadda $src1_dst, $src1_dst, $src2\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fadda(as_FloatRegister($src1_dst$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector max reduction

instruct reduce_maxF(vRegF dst, vRegF src1, vReg src2) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16);
  match(Set dst (MaxReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_fmaxv $dst, $src2 # vector (sve) (S)\n\t"
            "fmaxs $dst, $dst, $src1\t # max reduction F" %}
  ins_encode %{
    __ sve_fmaxv(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
    __ fmaxs(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_maxD(vRegD dst, vRegD src1, vReg src2) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16);
  match(Set dst (MaxReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_fmaxv $dst, $src2 # vector (sve) (S)\n\t"
            "fmaxs $dst, $dst, $src1\t # max reduction D" %}
  ins_encode %{
    __ sve_fmaxv(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
    __ fmaxd(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector min reduction

instruct reduce_minF(vRegF dst, vRegF src1, vReg src2) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_FLOAT &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16);
  match(Set dst (MinReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_fminv $dst, $src2 # vector (sve) (S)\n\t"
            "fmins $dst, $dst, $src1\t # min reduction F" %}
  ins_encode %{
    __ sve_fminv(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src2$$reg));
    __ fmins(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct reduce_minD(vRegD dst, vRegD src1, vReg src2) %{
  predicate(UseSVE > 0 && n->in(2)->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE &&
            n->in(2)->bottom_type()->is_vect()->length_in_bytes() >= 16);
  match(Set dst (MinReductionV src1 src2));
  ins_cost(INSN_COST);
  effect(TEMP_DEF dst);
  format %{ "sve_fminv $dst, $src2 # vector (sve) (S)\n\t"
            "fmins $dst, $dst, $src1\t # min reduction D" %}
  ins_encode %{
    __ sve_fminv(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src2$$reg));
    __ fmind(as_FloatRegister($dst$$reg), as_FloatRegister($dst$$reg), as_FloatRegister($src1$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector Math.rint, floor, ceil

instruct vroundD(vReg dst, vReg src, immI rmode) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2 &&
            n->bottom_type()->is_vect()->element_basic_type() == T_DOUBLE);
  match(Set dst (RoundDoubleModeV src rmode));
  format %{ "sve_frint $dst, $src, $rmode\t# vector (sve) (D)" %}
  ins_encode %{
    switch ($rmode$$constant) {
      case RoundDoubleModeNode::rmode_rint:
        __ sve_frintn(as_FloatRegister($dst$$reg), __ D,
             ptrue, as_FloatRegister($src$$reg));
        break;
      case RoundDoubleModeNode::rmode_floor:
        __ sve_frintm(as_FloatRegister($dst$$reg), __ D,
             ptrue, as_FloatRegister($src$$reg));
        break;
      case RoundDoubleModeNode::rmode_ceil:
        __ sve_frintp(as_FloatRegister($dst$$reg), __ D,
             ptrue, as_FloatRegister($src$$reg));
        break;
    }
  %}
  ins_pipe(pipe_slow);
%}

// vector replicate

instruct replicateB(vReg dst, iRegIorL2I src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (ReplicateB src));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $src\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateS(vReg dst, iRegIorL2I src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (ReplicateS src));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $src\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateI(vReg dst, iRegIorL2I src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (ReplicateI src));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateL(vReg dst, iRegL src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (ReplicateL src));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateB_imm8(vReg dst, immI8 con) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (ReplicateB con));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $con\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ B, $con$$constant);
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateS_imm8(vReg dst, immI8_shift8 con) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (ReplicateS con));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $con\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ H, $con$$constant);
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateI_imm8(vReg dst, immI8_shift8 con) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (ReplicateI con));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $con\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ S, $con$$constant);
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateL_imm8(vReg dst, immL8_shift8 con) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (ReplicateL con));
  ins_cost(SVE_COST);
  format %{ "sve_dup  $dst, $con\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ D, $con$$constant);
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateF(vReg dst, vRegF src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (ReplicateF src));
  ins_cost(SVE_COST);
  format %{ "sve_cpy  $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_cpy(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct replicateD(vReg dst, vRegD src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (ReplicateD src));
  ins_cost(SVE_COST);
  format %{ "sve_cpy  $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_cpy(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector shift

instruct vasrB(vReg dst, vReg shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (RShiftVB dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst$$reg), __ B,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrS(vReg dst, vReg shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (RShiftVS dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst$$reg), __ H,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrI(vReg dst, vReg shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (RShiftVI dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrL(vReg dst, vReg shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (RShiftVL dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $dst, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_asr(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslB(vReg dst, vReg shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (LShiftVB dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslS(vReg dst, vReg shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (LShiftVS dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslI(vReg dst, vReg shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (LShiftVI dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslL(vReg dst, vReg shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (LShiftVL dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $dst, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrB(vReg dst, vReg shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (URShiftVB dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrS(vReg dst, vReg shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (URShiftVS dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrI(vReg dst, vReg shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (URShiftVI dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrL(vReg dst, vReg shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (URShiftVL dst shift));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $dst, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($shift$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrB_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (RShiftVB src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    if (con >= 8) con = 7;
    __ sve_asr(as_FloatRegister($dst$$reg), __ B,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrS_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (RShiftVS src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    if (con >= 16) con = 15;
    __ sve_asr(as_FloatRegister($dst$$reg), __ H,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrI_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (RShiftVI src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_asr(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vasrL_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (RShiftVL src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_asr $dst, $src, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_asr(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrB_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (URShiftVB src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    if (con >= 8) {
      __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsr(as_FloatRegister($dst$$reg), __ B,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrS_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (URShiftVS src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    if (con >= 16) {
      __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsr(as_FloatRegister($dst$$reg), __ H,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrI_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (URShiftVI src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsr(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlsrL_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (URShiftVL src (RShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsr $dst, $src, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con == 0) {
      __ sve_orr(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsr(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslB_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (LShiftVB src (LShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (B)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con >= 8) {
      __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsl(as_FloatRegister($dst$$reg), __ B,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslS_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (LShiftVS src (LShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (H)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    if (con >= 16) {
      __ sve_eor(as_FloatRegister($dst$$reg), as_FloatRegister($src$$reg),
           as_FloatRegister($src$$reg));
      return;
    }
    __ sve_lsl(as_FloatRegister($dst$$reg), __ H,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslI_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (LShiftVI src (LShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (S)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    __ sve_lsl(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vlslL_imm(vReg dst, vReg src, immI shift) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (LShiftVL src (LShiftCntV shift)));
  ins_cost(SVE_COST);
  format %{ "sve_lsl $dst, $src, $shift\t# vector (sve) (D)" %}
  ins_encode %{
    int con = (int)$shift$$constant;
    __ sve_lsl(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src$$reg), con);
  %}
  ins_pipe(pipe_slow);
%}

instruct vshiftcntB(vReg dst, iRegIorL2I cnt) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_BYTE));
  match(Set dst (LShiftCntV cnt));
  match(Set dst (RShiftCntV cnt));
  format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (B)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ B, as_Register($cnt$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vshiftcntS(vReg dst, iRegIorL2I cnt) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_SHORT ||
            (n->bottom_type()->is_vect()->element_basic_type() == T_CHAR)));
  match(Set dst (LShiftCntV cnt));
  match(Set dst (RShiftCntV cnt));
  format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (H)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ H, as_Register($cnt$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vshiftcntI(vReg dst, iRegIorL2I cnt) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_INT));
  match(Set dst (LShiftCntV cnt));
  match(Set dst (RShiftCntV cnt));
  format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (S)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ S, as_Register($cnt$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vshiftcntL(vReg dst, iRegIorL2I cnt) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2 &&
            (n->bottom_type()->is_vect()->element_basic_type() == T_LONG));
  match(Set dst (LShiftCntV cnt));
  match(Set dst (RShiftCntV cnt));
  format %{ "sve_dup $dst, $cnt\t# vector shift count (sve) (D)" %}
  ins_encode %{
    __ sve_dup(as_FloatRegister($dst$$reg), __ D, as_Register($cnt$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector sqrt

instruct vsqrtF(vReg dst, vReg src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (SqrtVF src));
  ins_cost(SVE_COST);
  format %{ "sve_fsqrt $dst, $src\t# vector (sve) (S)" %}
  ins_encode %{
    __ sve_fsqrt(as_FloatRegister($dst$$reg), __ S,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsqrtD(vReg dst, vReg src) %{
  predicate(UseSVE > 0 && n->as_Vector()->length_in_bytes() >= 16);
  match(Set dst (SqrtVD src));
  ins_cost(SVE_COST);
  format %{ "sve_fsqrt $dst, $src\t# vector (sve) (D)" %}
  ins_encode %{
    __ sve_fsqrt(as_FloatRegister($dst$$reg), __ D,
         ptrue, as_FloatRegister($src$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector sub

instruct vsubB(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 16);
  match(Set dst (SubVB src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (B)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst$$reg), __ B,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubS(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 8);
  match(Set dst (SubVS src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (H)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst$$reg), __ H,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubI(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (SubVI src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubL(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (SubVL src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_sub $dst, $src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_sub(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubF(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 4);
  match(Set dst (SubVF src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (S)" %}
  ins_encode %{
    __ sve_fsub(as_FloatRegister($dst$$reg), __ S,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

instruct vsubD(vReg dst, vReg src1, vReg src2) %{
  predicate(UseSVE > 0 && n->as_Vector()->length() >= 2);
  match(Set dst (SubVD src1 src2));
  ins_cost(SVE_COST);
  format %{ "sve_fsub $dst, $src1, $src2\t # vector (sve) (D)" %}
  ins_encode %{
    __ sve_fsub(as_FloatRegister($dst$$reg), __ D,
         as_FloatRegister($src1$$reg),
         as_FloatRegister($src2$$reg));
  %}
  ins_pipe(pipe_slow);
%}

// vector mask cast

instruct vmaskcast(vReg dst) %{
  predicate(UseSVE > 0 && n->bottom_type()->is_vect()->length() == n->in(1)->bottom_type()->is_vect()->length() &&
            n->bottom_type()->is_vect()->length_in_bytes() == n->in(1)->bottom_type()->is_vect()->length_in_bytes());
  match(Set dst (VectorMaskCast dst));
  ins_cost(0);
  format %{ "vmaskcast $dst\t# empty (sve)" %}
  ins_encode %{
    // empty
  %}
  ins_pipe(pipe_class_empty);
%}