/* * Licensed to the Apache Software Foundation (ASF) under one * or more contributor license agreements. See the NOTICE file * distributed with this work for additional information * regarding copyright ownership. The ASF licenses this file * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, * software distributed under the License is distributed on an * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * KIND, either express or implied. See the License for the * specific language governing permissions and limitations * under the License. */ /*! * \file intrin_rule_default.cc * \brief Default intrinsic rules. */ #include "intrin_rule.h" #include #include namespace tvm { namespace codegen { namespace intrin { using tir::FLowerIntrinsic; TVM_REGISTER_OP("tir.exp").set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.erf").set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.log").set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.log2") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.log10") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.log1p") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.tanh") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.tan").set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.atan") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.atanh") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.atan2") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.cos").set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.acos") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.cosh") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.acosh") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.sin").set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.asin") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.sinh") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.asinh") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.hypot") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.nextafter") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.copysign") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.ldexp") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.sqrt") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.floor") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.ceil") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.round") .set_attr("default.FLowerIntrinsic", DispatchPureExtern); TVM_REGISTER_OP("tir.pow").set_attr("default.FLowerIntrinsic", DispatchPureExtern); } // namespace intrin namespace legalize { using namespace tir; TVM_REGISTER_OP("tir.rsqrt") .set_attr("default.FLegalize", [](const PrimExpr& e) -> PrimExpr { const CallNode* call = e.as(); ICHECK(call != nullptr); auto one = make_const(call->args[0].dtype(), 1); return one / sqrt(call->args[0]); }); TVM_REGISTER_OP("tir.sigmoid") .set_attr("default.FLegalize", [](const PrimExpr& e) -> PrimExpr { const CallNode* call = e.as(); ICHECK(call != nullptr); auto one = make_const(call->args[0].dtype(), 1); return one / (one + exp(-call->args[0])); }); TVM_REGISTER_OP("tir.isfinite") .set_attr("default.FLegalize", [](const PrimExpr& e) -> PrimExpr { const CallNode* call = e.as(); ICHECK(call != nullptr); return isfinite(call->args[0]); }); TVM_REGISTER_OP("tir.isinf") .set_attr("default.FLegalize", [](const PrimExpr& e) -> PrimExpr { const CallNode* call = e.as(); ICHECK(call != nullptr); return isinf(call->args[0]); }); TVM_REGISTER_OP("tir.q_multiply_shift") .set_attr("default.FLegalize", [](const PrimExpr& e) -> PrimExpr { using tir::make_const; const tir::CallNode* call = e.as(); ICHECK(call != nullptr); PrimExpr x = call->args[0]; PrimExpr y = call->args[1]; PrimExpr q = call->args[2]; PrimExpr s = call->args[3]; // Lambda function to extract the int value from PrimExpr auto get_int_value = [](const PrimExpr node) { if (auto int_node = node.as()) { return int_node->value; } auto broadcast_node = node.as(); CHECK(broadcast_node != nullptr); auto int_node = broadcast_node->value.as(); CHECK(int_node != nullptr); return int_node->value; }; // Power of 2 is determined by the fixed_point_multiplier == 1 << 30. In case of power of // 2, fixed point multiplier will represent a float value of 0.5. In fixed point, this is // represented by 1 << 30. if (get_int_value(y) == (1 << 30)) { PrimExpr exp = s - 1; int exp_val = get_int_value(s) - 1; if (exp_val > 0) { // power of 2 is greater than 0, apply left shift. return x << exp; } else { // power of 2 is less than 0, round and then apply right shift. DataType lp_dtype = DataType::Int(32, x.dtype().lanes()); PrimExpr one = make_const(lp_dtype, 1); exp = -exp; PrimExpr rounding_factor = one << (exp - 1); PrimExpr rounded_t = x + rounding_factor; return rounded_t >> exp; } } else { // Only int32 types are supported (any number of lanes is allowed) ICHECK(y.dtype().code() == DLDataTypeCode::kDLInt && y.dtype().bits() == 32); ICHECK(s.dtype().code() == DLDataTypeCode::kDLInt && s.dtype().bits() == 32); DataType hp_dtype = DataType::Int(64, x.dtype().lanes()); DataType lp_dtype = DataType::Int(32, x.dtype().lanes()); // 1) Calculating the integer multiplier and integer shift PrimExpr zero = make_const(s.dtype(), 0); PrimExpr left_shift = tir::Select(s > zero, s, zero); PrimExpr right_shift = tir::Select(s > zero, zero, -s); // 2) Cast and Multiply the integer multiplier PrimExpr one = make_const(hp_dtype, 1); x = cast(hp_dtype, x); y = cast(hp_dtype, y); x = tir::Select(left_shift != zero, x << left_shift, x); // 3) Perform the multiplication in higher precision. x = x * y; // 4) Find the rounding scalar PrimExpr total_right_shift = right_shift + q; PrimExpr pos_rounding_value = (one << (total_right_shift - 1)); x = x + pos_rounding_value; // 5) Simply right shift the result to get the final output. x = x >> total_right_shift; // 6) The fixed point multiplication keeps the value in int32 range. Casting back to // int32. return cast(lp_dtype, x); } }); } // namespace legalize } // namespace codegen } // namespace tvm