/*
 * Copyright (c) 2013-2015 Intel Corporation, Inc.  All rights reserved.
 * Copyright (c) 2017-2020 Amazon.com, Inc. or its affiliates. All rights reserved.
 *
 * This software is available to you under a choice of one of two
 * licenses.  You may choose to be licensed under the terms of the GNU
 * General Public License (GPL) Version 2, available from the file
 * COPYING in the main directory of this source tree, or the
 * BSD license below:
 *
 *     Redistribution and use in source and binary forms, with or
 *     without modification, are permitted provided that the following
 *     conditions are met:
 *
 *      - Redistributions of source code must retain the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer.
 *
 *      - Redistributions in binary form must reproduce the above
 *        copyright notice, this list of conditions and the following
 *        disclaimer in the documentation and/or other materials
 *        provided with the distribution.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 * SOFTWARE.
 */

#include "config.h"
#include "efa.h"

#include <sys/time.h>
#include <infiniband/efadv.h>
#define EFA_CQ_PROGRESS_ENTRIES 500

static int efa_generate_qkey()
{
	struct timeval tv;
	struct timezone tz;
	uint32_t val;
	int err;

	err = gettimeofday(&tv, &tz);
	if (err) {
		EFA_WARN(FI_LOG_EP_CTRL, "Cannot gettimeofday, err=%d.\n", err);
		return 0;
	}

	/* tv_usec is in range [0,1,000,000), shift it by 12 to [0,4,096,000,000 */
	val = (tv.tv_usec << 12) + tv.tv_sec;

	val = ofi_xorshift_random(val);

	/* 0x80000000 and up is privileged Q Key range. */
	val &= 0x7fffffff;

	return val;
}

static int efa_ep_destroy_qp(struct efa_qp *qp)
{
	struct efa_domain *domain;
	int err;

	if (!qp)
		return 0;

	domain = qp->ep->domain;
	domain->qp_table[qp->qp_num & domain->qp_table_sz_m1] = NULL;
	err = -ibv_destroy_qp(qp->ibv_qp);
	if (err)
		EFA_INFO(FI_LOG_CORE, "destroy qp[%u] failed!\n", qp->qp_num);

	free(qp);
	return err;
}

static int efa_ep_modify_qp_state(struct efa_qp *qp, enum ibv_qp_state qp_state,
				  int attr_mask)
{
	struct ibv_qp_attr attr = {};

	attr.qp_state = qp_state;

	if (attr_mask & IBV_QP_PORT)
		attr.port_num = 1;

	if (attr_mask & IBV_QP_QKEY)
		attr.qkey = qp->qkey;

	return -ibv_modify_qp(qp->ibv_qp, &attr, attr_mask);

}

static int efa_ep_modify_qp_rst2rts(struct efa_qp *qp)
{
	int err;

	err = efa_ep_modify_qp_state(qp, IBV_QPS_INIT,
				     IBV_QP_STATE | IBV_QP_PKEY_INDEX |
				     IBV_QP_PORT | IBV_QP_QKEY);
	if (err)
		return err;

	err = efa_ep_modify_qp_state(qp, IBV_QPS_RTR, IBV_QP_STATE);
	if (err)
		return err;

	return efa_ep_modify_qp_state(qp, IBV_QPS_RTS,
				      IBV_QP_STATE | IBV_QP_SQ_PSN);
}

static int efa_ep_create_qp_ex(struct efa_ep *ep,
			       struct ibv_pd *ibv_pd,
			       struct ibv_qp_init_attr_ex *init_attr_ex)
{
	struct efa_domain *domain;
	struct efa_qp *qp;
	struct efadv_qp_init_attr efa_attr = {};
	int err;

	domain = ep->domain;
	qp = calloc(1, sizeof(*qp));
	if (!qp)
		return -FI_ENOMEM;

	if (init_attr_ex->qp_type == IBV_QPT_UD) {
		qp->ibv_qp = ibv_create_qp_ex(ibv_pd->context, init_attr_ex);
	} else {
		assert(init_attr_ex->qp_type == IBV_QPT_DRIVER);
		efa_attr.driver_qp_type = EFADV_QP_DRIVER_TYPE_SRD;
		qp->ibv_qp = efadv_create_qp_ex(ibv_pd->context, init_attr_ex, &efa_attr,
						sizeof(struct efadv_qp_init_attr));
	}

	if (!qp->ibv_qp) {
		EFA_WARN(FI_LOG_EP_CTRL, "ibv_create_qp failed\n");
		err = -EINVAL;
		goto err_free_qp;
	}

	qp->ibv_qp_ex = ibv_qp_to_qp_ex(qp->ibv_qp);
	qp->qkey = efa_generate_qkey();
	err = efa_ep_modify_qp_rst2rts(qp);
	if (err)
		goto err_destroy_qp;

	qp->qp_num = qp->ibv_qp->qp_num;
	ep->qp = qp;
	qp->ep = ep;
	domain->qp_table[ep->qp->qp_num & domain->qp_table_sz_m1] = ep->qp;
	EFA_INFO(FI_LOG_EP_CTRL, "%s(): create QP %d qkey: %d\n", __func__, qp->qp_num, qp->qkey);

	return 0;

err_destroy_qp:
	ibv_destroy_qp(qp->ibv_qp);
err_free_qp:
	free(qp);

	return err;
}

static int efa_ep_getopt(fid_t fid, int level, int optname,
			 void *optval, size_t *optlen)
{
	switch (level) {
	case FI_OPT_ENDPOINT:
		return -FI_ENOPROTOOPT;
	default:
		return -FI_ENOPROTOOPT;
	}
	return 0;
}

static int efa_ep_setopt(fid_t fid, int level, int optname, const void *optval, size_t optlen)
{
	switch (level) {
	case FI_OPT_ENDPOINT:
		return -FI_ENOPROTOOPT;
	default:
		return -FI_ENOPROTOOPT;
	}
	return 0;
}

static struct fi_ops_ep efa_ep_base_ops = {
	.size = sizeof(struct fi_ops_ep),
	.cancel = fi_no_cancel,
	.getopt = efa_ep_getopt,
	.setopt = efa_ep_setopt,
	.tx_ctx = fi_no_tx_ctx,
	.rx_ctx = fi_no_rx_ctx,
	.rx_size_left = fi_no_rx_size_left,
	.tx_size_left = fi_no_tx_size_left,
};

static struct efa_ep *efa_ep_alloc(struct fi_info *info)
{
	struct efa_ep *ep;

	ep = calloc(1, sizeof(*ep));
	if (!ep)
		return NULL;

	ep->info = fi_dupinfo(info);
	if (!ep->info)
		goto err;

	return ep;

err:
	free(ep);
	return NULL;
}

static void efa_ep_destroy(struct efa_ep *ep)
{
	efa_ep_destroy_qp(ep->qp);
	fi_freeinfo(ep->info);
	free(ep->src_addr);
	if (ofi_endpoint_close(&ep->util_ep))
		FI_WARN(&efa_prov, FI_LOG_EP_CTRL, "Unable to close util EP\n");
	free(ep);
}

static int efa_ep_close(fid_t fid)
{
	struct efa_ep *ep;

	ep = container_of(fid, struct efa_ep, util_ep.ep_fid.fid);

	ofi_bufpool_destroy(ep->recv_wr_pool);
	ofi_bufpool_destroy(ep->send_wr_pool);
	efa_ep_destroy(ep);

	return 0;
}

static int efa_ep_bind(struct fid *fid, struct fid *bfid, uint64_t flags)
{
	struct efa_ep *ep;
	struct efa_cq *cq;
	struct efa_av *av;
	struct util_eq *eq;
	struct util_cntr *cntr;
	int ret;

	ep = container_of(fid, struct efa_ep, util_ep.ep_fid.fid);
	ret = ofi_ep_bind_valid(&efa_prov, bfid, flags);
	if (ret)
		return ret;

	switch (bfid->fclass) {
	case FI_CLASS_CQ:
		if (flags & FI_SELECTIVE_COMPLETION) {
			EFA_WARN(FI_LOG_EP_CTRL,
				 "Endpoint cannot be bound with selective completion.\n");
			return -FI_EBADFLAGS;
		}

		/* Must bind a CQ to either RECV or SEND completions */
		if (!(flags & (FI_RECV | FI_TRANSMIT)))
			return -FI_EBADFLAGS;

		cq = container_of(bfid, struct efa_cq, util_cq.cq_fid);
		if (ep->domain != cq->domain)
			return -FI_EINVAL;

		ret = ofi_ep_bind_cq(&ep->util_ep, &cq->util_cq, flags);
		if (ret)
			return ret;

		if (flags & FI_RECV) {
			if (ep->rcq)
				return -EINVAL;
			ep->rcq = cq;
		}
		if (flags & FI_TRANSMIT) {
			if (ep->scq)
				return -EINVAL;
			ep->scq = cq;
		}
		break;
	case FI_CLASS_AV:
		av = container_of(bfid, struct efa_av, util_av.av_fid.fid);
		if (ep->domain != av->domain) {
			EFA_WARN(FI_LOG_EP_CTRL,
				 "Address vector doesn't belong to same domain as EP.\n");
			return -FI_EINVAL;
		}
		if (ep->av) {
			EFA_WARN(FI_LOG_EP_CTRL,
				 "Address vector already bound to EP.\n");
			return -FI_EINVAL;
		}
		ep->av = av;

		ep->av->ep = ep;
		break;
	case FI_CLASS_CNTR:
		cntr = container_of(bfid, struct util_cntr, cntr_fid.fid);

		ret = ofi_ep_bind_cntr(&ep->util_ep, cntr, flags);
		if (ret)
			return ret;
		break;
	case FI_CLASS_EQ:
		eq = container_of(bfid, struct util_eq, eq_fid.fid);

		ret = ofi_ep_bind_eq(&ep->util_ep, eq);
		if (ret)
			return ret;
		break;
	default:
		return -EINVAL;
	}

	return 0;
}

static int efa_ep_getflags(struct fid_ep *ep_fid, uint64_t *flags)
{
	struct efa_ep *ep = container_of(ep_fid, struct efa_ep, util_ep.ep_fid);
	struct fi_tx_attr *tx_attr = ep->info->tx_attr;
	struct fi_rx_attr *rx_attr = ep->info->rx_attr;

	if ((*flags & FI_TRANSMIT) && (*flags & FI_RECV)) {
		EFA_WARN(FI_LOG_EP_CTRL, "Both Tx/Rx flags cannot be specified\n");
		return -FI_EINVAL;
	} else if (tx_attr && (*flags & FI_TRANSMIT)) {
		*flags = tx_attr->op_flags;
	} else if (rx_attr && (*flags & FI_RECV)) {
		*flags = rx_attr->op_flags;
	} else {
		EFA_WARN(FI_LOG_EP_CTRL, "Tx/Rx flags not specified\n");
		return -FI_EINVAL;
	}
	return 0;
}

static int efa_ep_setflags(struct fid_ep *ep_fid, uint64_t flags)
{
	struct efa_ep *ep = container_of(ep_fid, struct efa_ep, util_ep.ep_fid);
	struct fi_tx_attr *tx_attr = ep->info->tx_attr;
	struct fi_rx_attr *rx_attr = ep->info->rx_attr;

	if ((flags & FI_TRANSMIT) && (flags & FI_RECV)) {
		EFA_WARN(FI_LOG_EP_CTRL, "Both Tx/Rx flags cannot be specified.\n");
		return -FI_EINVAL;
	} else if (tx_attr && (flags & FI_TRANSMIT)) {
		tx_attr->op_flags = flags;
		tx_attr->op_flags &= ~FI_TRANSMIT;
	} else if (rx_attr && (flags & FI_RECV)) {
		rx_attr->op_flags = flags;
		rx_attr->op_flags &= ~FI_RECV;
	} else {
		EFA_WARN(FI_LOG_EP_CTRL, "Tx/Rx flags not specified\n");
		return -FI_EINVAL;
	}

	return 0;
}

static int efa_ep_enable(struct fid_ep *ep_fid)
{
	struct ibv_qp_init_attr_ex attr_ex = { 0 };
	const struct fi_info *efa_info;
	struct ibv_pd *ibv_pd;
	struct efa_ep *ep;

	ep = container_of(ep_fid, struct efa_ep, util_ep.ep_fid);

	if (!ep->scq && !ep->rcq) {
		EFA_WARN(FI_LOG_EP_CTRL,
			 "Endpoint is not bound to a send or receive completion queue\n");
		return -FI_ENOCQ;
	}

	if (!ep->scq && ofi_send_allowed(ep->info->caps)) {
		EFA_WARN(FI_LOG_EP_CTRL,
			 "Endpoint is not bound to a send completion queue when it has transmit capabilities enabled (FI_SEND).\n");
		return -FI_ENOCQ;
	}

	if (!ep->rcq && ofi_recv_allowed(ep->info->caps)) {
		EFA_WARN(FI_LOG_EP_CTRL,
			 "Endpoint is not bound to a receive completion queue when it has receive capabilities enabled. (FI_RECV)\n");
		return -FI_ENOCQ;
	}

	efa_info = efa_get_efa_info(ep->info->domain_attr->name);
	if (!efa_info) {
		EFA_INFO(FI_LOG_EP_CTRL, "Unable to find matching efa_info\n");
		return -FI_EINVAL;
	}

	if (ep->scq) {
		attr_ex.cap.max_send_wr = ep->info->tx_attr->size;
		attr_ex.cap.max_send_sge = ep->info->tx_attr->iov_limit;
		attr_ex.send_cq = ep->scq->ibv_cq;
		ibv_pd = ep->scq->domain->ibv_pd;
	} else {
		attr_ex.send_cq = ep->rcq->ibv_cq;
		ibv_pd = ep->rcq->domain->ibv_pd;
	}

	if (ep->rcq) {
		attr_ex.cap.max_recv_wr = ep->info->rx_attr->size;
		attr_ex.cap.max_recv_sge = ep->info->rx_attr->iov_limit;
		attr_ex.recv_cq = ep->rcq->ibv_cq;
	} else {
		attr_ex.recv_cq = ep->scq->ibv_cq;
	}

	attr_ex.cap.max_inline_data = ep->domain->ctx->inline_buf_size;

	if (ep->domain->type == EFA_DOMAIN_RDM) {
		attr_ex.qp_type = IBV_QPT_DRIVER;
		attr_ex.comp_mask = IBV_QP_INIT_ATTR_PD | IBV_QP_INIT_ATTR_SEND_OPS_FLAGS;
		attr_ex.send_ops_flags = IBV_QP_EX_WITH_SEND;
		if (efa_ep_support_rdma_read(&ep->util_ep.ep_fid))
			attr_ex.send_ops_flags |= IBV_QP_EX_WITH_RDMA_READ;
		attr_ex.pd = ibv_pd;
	} else {
		attr_ex.qp_type = IBV_QPT_UD;
		attr_ex.comp_mask = IBV_QP_INIT_ATTR_PD;
		attr_ex.pd = ibv_pd;
	}

	attr_ex.qp_context = ep;
	attr_ex.sq_sig_all = 1;

	return efa_ep_create_qp_ex(ep, ibv_pd, &attr_ex);
}

static int efa_ep_control(struct fid *fid, int command, void *arg)
{
	struct fid_ep *ep_fid;

	switch (fid->fclass) {
	case FI_CLASS_EP:
		ep_fid = container_of(fid, struct fid_ep, fid);
		switch (command) {
		case FI_GETOPSFLAG:
			return efa_ep_getflags(ep_fid, (uint64_t *)arg);
		case FI_SETOPSFLAG:
			return efa_ep_setflags(ep_fid, *(uint64_t *)arg);
		case FI_ENABLE:
			return efa_ep_enable(ep_fid);
		default:
			return -FI_ENOSYS;
		}
		break;
	default:
		return -FI_ENOSYS;
	}
}

static struct fi_ops efa_ep_ops = {
	.size = sizeof(struct fi_ops),
	.close = efa_ep_close,
	.bind = efa_ep_bind,
	.control = efa_ep_control,
	.ops_open = fi_no_ops_open,
};

static void efa_ep_progress_internal(struct efa_ep *ep, struct efa_cq *efa_cq)
{
	struct util_cq *cq;
	struct fi_cq_tagged_entry cq_entry[EFA_CQ_PROGRESS_ENTRIES];
	struct fi_cq_tagged_entry *temp_cq_entry;
	struct fi_cq_err_entry cq_err_entry;
	fi_addr_t src_addr[EFA_CQ_PROGRESS_ENTRIES];
	uint64_t flags;
	int i;
	ssize_t ret, err;

	cq = &efa_cq->util_cq;
	flags = ep->util_ep.caps;

	VALGRIND_MAKE_MEM_DEFINED(&cq_entry, sizeof(cq_entry));

	ret = efa_cq_readfrom(&cq->cq_fid, cq_entry, EFA_CQ_PROGRESS_ENTRIES,
			      (flags & FI_SOURCE) ? src_addr : NULL);
	if (ret == -FI_EAGAIN)
		return;

	if (OFI_UNLIKELY(ret < 0)) {
		if (OFI_UNLIKELY(ret != -FI_EAVAIL)) {
			EFA_WARN(FI_LOG_CQ, "no error available errno: %ld\n", ret);
			efa_eq_write_error(&ep->util_ep, FI_EOTHER, ret);
			return;
		}

		err = efa_cq_readerr(&cq->cq_fid, &cq_err_entry, flags);
		if (OFI_UNLIKELY(err < 0)) {
			EFA_WARN(FI_LOG_CQ, "unable to read error entry errno: %ld\n", err);
			efa_eq_write_error(&ep->util_ep, FI_EOTHER, err);
			return;
		}

		ofi_cq_write_error(cq, &cq_err_entry);
		return;
	}

	temp_cq_entry = (struct fi_cq_tagged_entry *)cq_entry;
	for (i = 0; i < ret; i++) {
		(flags & FI_SOURCE) ?
			ofi_cq_write_src(cq, temp_cq_entry->op_context,
					 temp_cq_entry->flags,
					 temp_cq_entry->len,
					 temp_cq_entry->buf,
					 temp_cq_entry->data,
					 temp_cq_entry->tag,
					 src_addr[i]) :
			ofi_cq_write(cq, temp_cq_entry->op_context,
				     temp_cq_entry->flags,
				     temp_cq_entry->len,
				     temp_cq_entry->buf,
				     temp_cq_entry->data,
				     temp_cq_entry->tag);

		temp_cq_entry = (struct fi_cq_tagged_entry *)
				((uint8_t *)temp_cq_entry + efa_cq->entry_size);
	}
	return;
}

void efa_ep_progress(struct util_ep *ep)
{
	struct efa_ep *efa_ep;
	struct efa_cq *rcq;
	struct efa_cq *scq;

	efa_ep = container_of(ep, struct efa_ep, util_ep);
	rcq = efa_ep->rcq;
	scq = efa_ep->scq;

	fastlock_acquire(&ep->lock);

	if (rcq)
		efa_ep_progress_internal(efa_ep, rcq);

	if (scq && scq != rcq)
		efa_ep_progress_internal(efa_ep, scq);

	fastlock_release(&ep->lock);
}

static struct fi_ops_atomic efa_ep_atomic_ops = {
	.size = sizeof(struct fi_ops_atomic),
	.write = fi_no_atomic_write,
	.writev = fi_no_atomic_writev,
	.writemsg = fi_no_atomic_writemsg,
	.inject = fi_no_atomic_inject,
	.readwrite = fi_no_atomic_readwrite,
	.readwritev = fi_no_atomic_readwritev,
	.readwritemsg = fi_no_atomic_readwritemsg,
	.compwrite = fi_no_atomic_compwrite,
	.compwritev = fi_no_atomic_compwritev,
	.compwritemsg = fi_no_atomic_compwritemsg,
	.writevalid = fi_no_atomic_writevalid,
	.readwritevalid = fi_no_atomic_readwritevalid,
	.compwritevalid = fi_no_atomic_compwritevalid,
};

int efa_ep_open(struct fid_domain *domain_fid, struct fi_info *info,
		struct fid_ep **ep_fid, void *context)
{
	struct efa_domain *domain;
	const struct fi_info *fi;
	struct efa_ep *ep;
	int ret;

	domain = container_of(domain_fid, struct efa_domain,
			      util_domain.domain_fid);

	if (!info || !info->ep_attr || !info->domain_attr ||
	    strncmp(domain->ctx->ibv_ctx->device->name, info->domain_attr->name,
		    strlen(domain->ctx->ibv_ctx->device->name))) {
		EFA_INFO(FI_LOG_DOMAIN, "Invalid info->domain_attr->name\n");
		return -FI_EINVAL;
	}

	fi = efa_get_efa_info(info->domain_attr->name);
	if (!fi) {
		EFA_INFO(FI_LOG_DOMAIN, "Unable to find matching efa_info\n");
		return -FI_EINVAL;
	}

	if (info->ep_attr) {
		ret = ofi_check_ep_attr(&efa_util_prov, info->fabric_attr->api_version, fi, info);
		if (ret)
			return ret;
	}

	if (info->tx_attr) {
		ret = ofi_check_tx_attr(&efa_prov, fi->tx_attr,
					info->tx_attr, info->mode);
		if (ret)
			return ret;
	}

	if (info->rx_attr) {
		ret = ofi_check_rx_attr(&efa_prov, fi, info->rx_attr, info->mode);
		if (ret)
			return ret;
	}

	ep = efa_ep_alloc(info);
	if (!ep)
		return -FI_ENOMEM;

	ret = ofi_endpoint_init(domain_fid, &efa_util_prov, info, &ep->util_ep,
				context, efa_ep_progress);
	if (ret)
		goto err_ep_destroy;

	ret = ofi_bufpool_create(&ep->send_wr_pool,
		sizeof(struct efa_send_wr) +
		info->tx_attr->iov_limit * sizeof(struct ibv_sge),
		16, 0, 1024, 0);
	if (ret)
		goto err_ep_destroy;

	ret = ofi_bufpool_create(&ep->recv_wr_pool,
		sizeof(struct efa_recv_wr) +
		info->rx_attr->iov_limit * sizeof(struct ibv_sge),
		16, 0, 1024, 0);
	if (ret)
		goto err_send_wr_destroy;

	ep->domain = domain;
	ep->xmit_more_wr_tail = &ep->xmit_more_wr_head;
	ep->recv_more_wr_tail = &ep->recv_more_wr_head;

	if (info->src_addr) {
		ep->src_addr = (void *)calloc(1, EFA_EP_ADDR_LEN);
		if (!ep->src_addr) {
			ret = -FI_ENOMEM;
			goto err_recv_wr_destroy;
		}
		memcpy(ep->src_addr, info->src_addr, info->src_addrlen);
	}

	*ep_fid = &ep->util_ep.ep_fid;
	(*ep_fid)->fid.fclass = FI_CLASS_EP;
	(*ep_fid)->fid.context = context;
	(*ep_fid)->fid.ops = &efa_ep_ops;
	(*ep_fid)->ops = &efa_ep_base_ops;
	(*ep_fid)->msg = &efa_ep_msg_ops;
	(*ep_fid)->cm = &efa_ep_cm_ops;
	(*ep_fid)->rma = &efa_ep_rma_ops;
	(*ep_fid)->atomic = &efa_ep_atomic_ops;

	return 0;

err_recv_wr_destroy:
	ofi_bufpool_destroy(ep->recv_wr_pool);
err_send_wr_destroy:
	ofi_bufpool_destroy(ep->send_wr_pool);
err_ep_destroy:
	efa_ep_destroy(ep);
	return ret;
}