drivers/infiniband/hw/hns/hns_roce_eq.c - nest-learning-thermostat/6.0/linux-imx-4.9.88 - Git at Google

 /*
  * Copyright (c) 2016 Hisilicon Limited.
  *
  * This software is available to you under a choice of one of two
  * licenses.  You may choose to be licensed under the terms of the GNU
  * General Public License (GPL) Version 2, available from the file
  * COPYING in the main directory of this source tree, or the
  * OpenIB.org BSD license below:
  *
  *     Redistribution and use in source and binary forms, with or
  *     without modification, are permitted provided that the following
  *     conditions are met:
  *
  *      - Redistributions of source code must retain the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer.
  *
  *      - Redistributions in binary form must reproduce the above
  *        copyright notice, this list of conditions and the following
  *        disclaimer in the documentation and/or other materials
  *        provided with the distribution.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  * SOFTWARE.
  */

 #include <linux/platform_device.h>
 #include "hns_roce_common.h"
 #include "hns_roce_device.h"
 #include "hns_roce_eq.h"

 static void eq_set_cons_index(struct hns_roce_eq *eq, int req_not)
 {
 	roce_raw_write((eq->cons_index & CONS_INDEX_MASK) |
 		      (req_not << eq->log_entries), eq->doorbell);
 	/* Memory barrier */
 	mb();
 }

 static struct hns_roce_aeqe *get_aeqe(struct hns_roce_eq *eq, u32 entry)
 {
 	unsigned long off = (entry & (eq->entries - 1)) *
 			     HNS_ROCE_AEQ_ENTRY_SIZE;

 	return (struct hns_roce_aeqe *)((u8 *)
 		(eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) +
 		off % HNS_ROCE_BA_SIZE);
 }

 static struct hns_roce_aeqe *next_aeqe_sw(struct hns_roce_eq *eq)
 {
 	struct hns_roce_aeqe *aeqe = get_aeqe(eq, eq->cons_index);

 	return (roce_get_bit(aeqe->asyn, HNS_ROCE_AEQE_U32_4_OWNER_S) ^
 		!!(eq->cons_index & eq->entries)) ? aeqe : NULL;
 }

 static void hns_roce_wq_catas_err_handle(struct hns_roce_dev *hr_dev,
 					 struct hns_roce_aeqe *aeqe, int qpn)
 {
 	struct device *dev = &hr_dev->pdev->dev;

 	dev_warn(dev, "Local Work Queue Catastrophic Error.\n");
 	switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
 			       HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
 	case HNS_ROCE_LWQCE_QPC_ERROR:
 		dev_warn(dev, "QP %d, QPC error.\n", qpn);
 		break;
 	case HNS_ROCE_LWQCE_MTU_ERROR:
 		dev_warn(dev, "QP %d, MTU error.\n", qpn);
 		break;
 	case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR:
 		dev_warn(dev, "QP %d, WQE BA addr error.\n", qpn);
 		break;
 	case HNS_ROCE_LWQCE_WQE_ADDR_ERROR:
 		dev_warn(dev, "QP %d, WQE addr error.\n", qpn);
 		break;
 	case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR:
 		dev_warn(dev, "QP %d, WQE shift error\n", qpn);
 		break;
 	case HNS_ROCE_LWQCE_SL_ERROR:
 		dev_warn(dev, "QP %d, SL error.\n", qpn);
 		break;
 	case HNS_ROCE_LWQCE_PORT_ERROR:
 		dev_warn(dev, "QP %d, port error.\n", qpn);
 		break;
 	default:
 		break;
 	}
 }

 static void hns_roce_local_wq_access_err_handle(struct hns_roce_dev *hr_dev,
 						struct hns_roce_aeqe *aeqe,
 						int qpn)
 {
 	struct device *dev = &hr_dev->pdev->dev;

 	dev_warn(dev, "Local Access Violation Work Queue Error.\n");
 	switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
 			       HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
 	case HNS_ROCE_LAVWQE_R_KEY_VIOLATION:
 		dev_warn(dev, "QP %d, R_key violation.\n", qpn);
 		break;
 	case HNS_ROCE_LAVWQE_LENGTH_ERROR:
 		dev_warn(dev, "QP %d, length error.\n", qpn);
 		break;
 	case HNS_ROCE_LAVWQE_VA_ERROR:
 		dev_warn(dev, "QP %d, VA error.\n", qpn);
 		break;
 	case HNS_ROCE_LAVWQE_PD_ERROR:
 		dev_err(dev, "QP %d, PD error.\n", qpn);
 		break;
 	case HNS_ROCE_LAVWQE_RW_ACC_ERROR:
 		dev_warn(dev, "QP %d, rw acc error.\n", qpn);
 		break;
 	case HNS_ROCE_LAVWQE_KEY_STATE_ERROR:
 		dev_warn(dev, "QP %d, key state error.\n", qpn);
 		break;
 	case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR:
 		dev_warn(dev, "QP %d, MR operation error.\n", qpn);
 		break;
 	default:
 		break;
 	}
 }

 static void hns_roce_qp_err_handle(struct hns_roce_dev *hr_dev,
 				   struct hns_roce_aeqe *aeqe,
 				   int event_type)
 {
 	struct device *dev = &hr_dev->pdev->dev;
 	int phy_port;
 	int qpn;

 	qpn = roce_get_field(aeqe->event.qp_event.qp,
 			     HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M,
 			     HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S);
 	phy_port = roce_get_field(aeqe->event.qp_event.qp,
 			HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M,
 			HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S);
 	if (qpn <= 1)
 		qpn = HNS_ROCE_MAX_PORTS * qpn + phy_port;

 	switch (event_type) {
 	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
 		dev_warn(dev, "Invalid Req Local Work Queue Error.\n"
 			      "QP %d, phy_port %d.\n", qpn, phy_port);
 		break;
 	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
 		hns_roce_wq_catas_err_handle(hr_dev, aeqe, qpn);
 		break;
 	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
 		hns_roce_local_wq_access_err_handle(hr_dev, aeqe, qpn);
 		break;
 	default:
 		break;
 	}

 	hns_roce_qp_event(hr_dev, qpn, event_type);
 }

 static void hns_roce_cq_err_handle(struct hns_roce_dev *hr_dev,
 				   struct hns_roce_aeqe *aeqe,
 				   int event_type)
 {
 	struct device *dev = &hr_dev->pdev->dev;
 	u32 cqn;

 	cqn = le32_to_cpu(roce_get_field(aeqe->event.cq_event.cq,
 		    HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M,
 		    HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S));

 	switch (event_type) {
 	case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
 		dev_warn(dev, "CQ 0x%x access err.\n", cqn);
 		break;
 	case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
 		dev_warn(dev, "CQ 0x%x overflow\n", cqn);
 		break;
 	case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID:
 		dev_warn(dev, "CQ 0x%x ID invalid.\n", cqn);
 		break;
 	default:
 		break;
 	}

 	hns_roce_cq_event(hr_dev, cqn, event_type);
 }

 static void hns_roce_db_overflow_handle(struct hns_roce_dev *hr_dev,
 					struct hns_roce_aeqe *aeqe)
 {
 	struct device *dev = &hr_dev->pdev->dev;

 	switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
 			       HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
 	case HNS_ROCE_DB_SUBTYPE_SDB_OVF:
 		dev_warn(dev, "SDB overflow.\n");
 		break;
 	case HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF:
 		dev_warn(dev, "SDB almost overflow.\n");
 		break;
 	case HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP:
 		dev_warn(dev, "SDB almost empty.\n");
 		break;
 	case HNS_ROCE_DB_SUBTYPE_ODB_OVF:
 		dev_warn(dev, "ODB overflow.\n");
 		break;
 	case HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF:
 		dev_warn(dev, "ODB almost overflow.\n");
 		break;
 	case HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP:
 		dev_warn(dev, "SDB almost empty.\n");
 		break;
 	default:
 		break;
 	}
 }

 static int hns_roce_aeq_int(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
 {
 	struct device *dev = &hr_dev->pdev->dev;
 	struct hns_roce_aeqe *aeqe;
 	int aeqes_found = 0;
 	int event_type;

 	while ((aeqe = next_aeqe_sw(eq))) {
 		dev_dbg(dev, "aeqe = %p, aeqe->asyn.event_type = 0x%lx\n", aeqe,
 			roce_get_field(aeqe->asyn,
 				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
 				       HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S));
 		/* Memory barrier */
 		rmb();

 		event_type = roce_get_field(aeqe->asyn,
 				HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
 				HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S);
 		switch (event_type) {
 		case HNS_ROCE_EVENT_TYPE_PATH_MIG:
 			dev_warn(dev, "PATH MIG not supported\n");
 			break;
 		case HNS_ROCE_EVENT_TYPE_COMM_EST:
 			dev_warn(dev, "COMMUNICATION established\n");
 			break;
 		case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
 			dev_warn(dev, "SQ DRAINED not supported\n");
 			break;
 		case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
 			dev_warn(dev, "PATH MIG failed\n");
 			break;
 		case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
 		case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
 		case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
 			hns_roce_qp_err_handle(hr_dev, aeqe, event_type);
 			break;
 		case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
 		case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
 		case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
 			dev_warn(dev, "SRQ not support!\n");
 			break;
 		case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
 		case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
 		case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID:
 			hns_roce_cq_err_handle(hr_dev, aeqe, event_type);
 			break;
 		case HNS_ROCE_EVENT_TYPE_PORT_CHANGE:
 			dev_warn(dev, "port change.\n");
 			break;
 		case HNS_ROCE_EVENT_TYPE_MB:
 			hns_roce_cmd_event(hr_dev,
 					   le16_to_cpu(aeqe->event.cmd.token),
 					   aeqe->event.cmd.status,
 					   le64_to_cpu(aeqe->event.cmd.out_param
 					   ));
 			break;
 		case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
 			hns_roce_db_overflow_handle(hr_dev, aeqe);
 			break;
 		case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
 			dev_warn(dev, "CEQ 0x%lx overflow.\n",
 			roce_get_field(aeqe->event.ce_event.ceqe,
 				     HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M,
 				     HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S));
 			break;
 		default:
 			dev_warn(dev, "Unhandled event %d on EQ %d at index %u\n",
 				 event_type, eq->eqn, eq->cons_index);
 			break;
 		};

 		eq->cons_index++;
 		aeqes_found = 1;

 		if (eq->cons_index > 2 * hr_dev->caps.aeqe_depth - 1) {
 			dev_warn(dev, "cons_index overflow, set back to zero\n"
 				);
 			eq->cons_index = 0;
 		}
 	}

 	eq_set_cons_index(eq, 0);

 	return aeqes_found;
 }

 static struct hns_roce_ceqe *get_ceqe(struct hns_roce_eq *eq, u32 entry)
 {
 	unsigned long off = (entry & (eq->entries - 1)) *
 			     HNS_ROCE_CEQ_ENTRY_SIZE;

 	return (struct hns_roce_ceqe *)((u8 *)
 			(eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) +
 			off % HNS_ROCE_BA_SIZE);
 }

 static struct hns_roce_ceqe *next_ceqe_sw(struct hns_roce_eq *eq)
 {
 	struct hns_roce_ceqe *ceqe = get_ceqe(eq, eq->cons_index);

 	return (!!(roce_get_bit(ceqe->ceqe.comp,
 		 HNS_ROCE_CEQE_CEQE_COMP_OWNER_S))) ^
 		 (!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
 }

 static int hns_roce_ceq_int(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
 {
 	struct hns_roce_ceqe *ceqe;
 	int ceqes_found = 0;
 	u32 cqn;

 	while ((ceqe = next_ceqe_sw(eq))) {
 		/* Memory barrier */
 		rmb();
 		cqn = roce_get_field(ceqe->ceqe.comp,
 				     HNS_ROCE_CEQE_CEQE_COMP_CQN_M,
 				     HNS_ROCE_CEQE_CEQE_COMP_CQN_S);
 		hns_roce_cq_completion(hr_dev, cqn);

 		++eq->cons_index;
 		ceqes_found = 1;

 		if (eq->cons_index > 2 * hr_dev->caps.ceqe_depth[eq->eqn] - 1) {
 			dev_warn(&eq->hr_dev->pdev->dev,
 				"cons_index overflow, set back to zero\n");
 			eq->cons_index = 0;
 		}
 	}

 	eq_set_cons_index(eq, 0);

 	return ceqes_found;
 }

 static int hns_roce_aeq_ovf_int(struct hns_roce_dev *hr_dev,
 				struct hns_roce_eq *eq)
 {
 	struct device *dev = &eq->hr_dev->pdev->dev;
 	int eqovf_found = 0;
 	u32 caepaemask_val;
 	u32 cealmovf_val;
 	u32 caepaest_val;
 	u32 aeshift_val;
 	u32 ceshift_val;
 	u32 cemask_val;
 	int i = 0;

 	/**
 	* AEQ overflow ECC mult bit err CEQ overflow alarm
 	* must clear interrupt, mask irq, clear irq, cancel mask operation
 	*/
 	aeshift_val = roce_read(hr_dev, ROCEE_CAEP_AEQC_AEQE_SHIFT_REG);

 	if (roce_get_bit(aeshift_val,
 		ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S) == 1) {
 		dev_warn(dev, "AEQ overflow!\n");

 		/* Set mask */
 		caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
 		roce_set_bit(caepaemask_val,
 			     ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
 			     HNS_ROCE_INT_MASK_ENABLE);
 		roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val);

 		/* Clear int state(INT_WC : write 1 clear) */
 		caepaest_val = roce_read(hr_dev, ROCEE_CAEP_AE_ST_REG);
 		roce_set_bit(caepaest_val,
 			     ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S, 1);
 		roce_write(hr_dev, ROCEE_CAEP_AE_ST_REG, caepaest_val);

 		/* Clear mask */
 		caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
 		roce_set_bit(caepaemask_val,
 			     ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
 			     HNS_ROCE_INT_MASK_DISABLE);
 		roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val);
 	}

 	/* CEQ almost overflow */
 	for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) {
 		ceshift_val = roce_read(hr_dev, ROCEE_CAEP_CEQC_SHIFT_0_REG +
 					i * CEQ_REG_OFFSET);

 		if (roce_get_bit(ceshift_val,
 		ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S) == 1) {
 			dev_warn(dev, "CEQ[%d] almost overflow!\n", i);
 			eqovf_found++;

 			/* Set mask */
 			cemask_val = roce_read(hr_dev,
 					       ROCEE_CAEP_CE_IRQ_MASK_0_REG +
 					       i * CEQ_REG_OFFSET);
 			roce_set_bit(cemask_val,
 				ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S,
 				HNS_ROCE_INT_MASK_ENABLE);
 			roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
 				   i * CEQ_REG_OFFSET, cemask_val);

 			/* Clear int state(INT_WC : write 1 clear) */
 			cealmovf_val = roce_read(hr_dev,
 				       ROCEE_CAEP_CEQ_ALM_OVF_0_REG +
 				       i * CEQ_REG_OFFSET);
 			roce_set_bit(cealmovf_val,
 				     ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S,
 				     1);
 			roce_write(hr_dev, ROCEE_CAEP_CEQ_ALM_OVF_0_REG +
 				    i * CEQ_REG_OFFSET, cealmovf_val);

 			/* Clear mask */
 			cemask_val = roce_read(hr_dev,
 				     ROCEE_CAEP_CE_IRQ_MASK_0_REG +
 				     i * CEQ_REG_OFFSET);
 			roce_set_bit(cemask_val,
 			       ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S,
 			       HNS_ROCE_INT_MASK_DISABLE);
 			roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
 				   i * CEQ_REG_OFFSET, cemask_val);
 		}
 	}

 	/* ECC multi-bit error alarm */
 	dev_warn(dev, "ECC UCERR ALARM: 0x%x, 0x%x, 0x%x\n",
 		 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM0_REG),
 		 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM1_REG),
 		 roce_read(hr_dev, ROCEE_ECC_UCERR_ALM2_REG));

 	dev_warn(dev, "ECC CERR ALARM: 0x%x, 0x%x, 0x%x\n",
 		 roce_read(hr_dev, ROCEE_ECC_CERR_ALM0_REG),
 		 roce_read(hr_dev, ROCEE_ECC_CERR_ALM1_REG),
 		 roce_read(hr_dev, ROCEE_ECC_CERR_ALM2_REG));

 	return eqovf_found;
 }

 static int hns_roce_eq_int(struct hns_roce_dev *hr_dev, struct hns_roce_eq *eq)
 {
 	int eqes_found = 0;

 	if (likely(eq->type_flag == HNS_ROCE_CEQ))
 		/* CEQ irq routine, CEQ is pulse irq, not clear */
 		eqes_found = hns_roce_ceq_int(hr_dev, eq);
 	else if (likely(eq->type_flag == HNS_ROCE_AEQ))
 		/* AEQ irq routine, AEQ is pulse irq, not clear */
 		eqes_found = hns_roce_aeq_int(hr_dev, eq);
 	else
 		/* AEQ queue overflow irq */
 		eqes_found = hns_roce_aeq_ovf_int(hr_dev, eq);

 	return eqes_found;
 }

 static irqreturn_t hns_roce_msi_x_interrupt(int irq, void *eq_ptr)
 {
 	int int_work = 0;
 	struct hns_roce_eq  *eq  = eq_ptr;
 	struct hns_roce_dev *hr_dev = eq->hr_dev;

 	int_work = hns_roce_eq_int(hr_dev, eq);

 	return IRQ_RETVAL(int_work);
 }

 static void hns_roce_enable_eq(struct hns_roce_dev *hr_dev, int eq_num,
 			       int enable_flag)
 {
 	void __iomem *eqc = hr_dev->eq_table.eqc_base[eq_num];
 	u32 val;

 	val = readl(eqc);

 	if (enable_flag)
 		roce_set_field(val,
 			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
 			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
 			       HNS_ROCE_EQ_STAT_VALID);
 	else
 		roce_set_field(val,
 			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
 			       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
 			       HNS_ROCE_EQ_STAT_INVALID);
 	writel(val, eqc);
 }

 static int hns_roce_create_eq(struct hns_roce_dev *hr_dev,
 			      struct hns_roce_eq *eq)
 {
 	void __iomem *eqc = hr_dev->eq_table.eqc_base[eq->eqn];
 	struct device *dev = &hr_dev->pdev->dev;
 	dma_addr_t tmp_dma_addr;
 	u32 eqconsindx_val = 0;
 	u32 eqcuridx_val = 0;
 	u32 eqshift_val = 0;
 	int num_bas = 0;
 	int ret;
 	int i;

 	num_bas = (PAGE_ALIGN(eq->entries * eq->eqe_size) +
 		   HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE;

 	if ((eq->entries * eq->eqe_size) > HNS_ROCE_BA_SIZE) {
 		dev_err(dev, "[error]eq buf %d gt ba size(%d) need bas=%d\n",
 			(eq->entries * eq->eqe_size), HNS_ROCE_BA_SIZE,
 			num_bas);
 		return -EINVAL;
 	}

 	eq->buf_list = kcalloc(num_bas, sizeof(*eq->buf_list), GFP_KERNEL);
 	if (!eq->buf_list)
 		return -ENOMEM;

 	for (i = 0; i < num_bas; ++i) {
 		eq->buf_list[i].buf = dma_alloc_coherent(dev, HNS_ROCE_BA_SIZE,
 							 &tmp_dma_addr,
 							 GFP_KERNEL);
 		if (!eq->buf_list[i].buf) {
 			ret = -ENOMEM;
 			goto err_out_free_pages;
 		}

 		eq->buf_list[i].map = tmp_dma_addr;
 		memset(eq->buf_list[i].buf, 0, HNS_ROCE_BA_SIZE);
 	}
 	eq->cons_index = 0;
 	roce_set_field(eqshift_val,
 		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
 		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
 		       HNS_ROCE_EQ_STAT_INVALID);
 	roce_set_field(eqshift_val,
 		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M,
 		       ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S,
 		       eq->log_entries);
 	writel(eqshift_val, eqc);

 	/* Configure eq extended address 12~44bit */
 	writel((u32)(eq->buf_list[0].map >> 12), (u8 *)eqc + 4);

 	/*
 	 * Configure eq extended address 45~49 bit.
 	 * 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
 	 * using 4K page, and shift more 32 because of
 	 * caculating the high 32 bit value evaluated to hardware.
 	 */
 	roce_set_field(eqcuridx_val, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M,
 		       ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S,
 		       eq->buf_list[0].map >> 44);
 	roce_set_field(eqcuridx_val,
 		       ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M,
 		       ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S, 0);
 	writel(eqcuridx_val, (u8 *)eqc + 8);

 	/* Configure eq consumer index */
 	roce_set_field(eqconsindx_val,
 		       ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M,
 		       ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S, 0);
 	writel(eqconsindx_val, (u8 *)eqc + 0xc);

 	return 0;

 err_out_free_pages:
 	for (i = i - 1; i >= 0; i--)
 		dma_free_coherent(dev, HNS_ROCE_BA_SIZE, eq->buf_list[i].buf,
 				  eq->buf_list[i].map);

 	kfree(eq->buf_list);
 	return ret;
 }

 static void hns_roce_free_eq(struct hns_roce_dev *hr_dev,
 			     struct hns_roce_eq *eq)
 {
 	int i = 0;
 	int npages = (PAGE_ALIGN(eq->eqe_size * eq->entries) +
 		      HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE;

 	if (!eq->buf_list)
 		return;

 	for (i = 0; i < npages; ++i)
 		dma_free_coherent(&hr_dev->pdev->dev, HNS_ROCE_BA_SIZE,
 				  eq->buf_list[i].buf, eq->buf_list[i].map);

 	kfree(eq->buf_list);
 }

 static void hns_roce_int_mask_en(struct hns_roce_dev *hr_dev)
 {
 	int i = 0;
 	u32 aemask_val;
 	int masken = 0;

 	/* AEQ INT */
 	aemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
 	roce_set_bit(aemask_val, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
 		     masken);
 	roce_set_bit(aemask_val, ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S, masken);
 	roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, aemask_val);

 	/* CEQ INT */
 	for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) {
 		/* IRQ mask */
 		roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
 			   i * CEQ_REG_OFFSET, masken);
 	}
 }

 static void hns_roce_ce_int_default_cfg(struct hns_roce_dev *hr_dev)
 {
 	/* Configure ce int interval */
 	roce_write(hr_dev, ROCEE_CAEP_CE_INTERVAL_CFG_REG,
 		   HNS_ROCE_CEQ_DEFAULT_INTERVAL);

 	/* Configure ce int burst num */
 	roce_write(hr_dev, ROCEE_CAEP_CE_BURST_NUM_CFG_REG,
 		   HNS_ROCE_CEQ_DEFAULT_BURST_NUM);
 }

 int hns_roce_init_eq_table(struct hns_roce_dev *hr_dev)
 {
 	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
 	struct device *dev = &hr_dev->pdev->dev;
 	struct hns_roce_eq *eq = NULL;
 	int eq_num = 0;
 	int ret = 0;
 	int i = 0;
 	int j = 0;

 	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
 	eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
 	if (!eq_table->eq)
 		return -ENOMEM;

 	eq_table->eqc_base = kcalloc(eq_num, sizeof(*eq_table->eqc_base),
 				     GFP_KERNEL);
 	if (!eq_table->eqc_base) {
 		ret = -ENOMEM;
 		goto err_eqc_base_alloc_fail;
 	}

 	for (i = 0; i < eq_num; i++) {
 		eq = &eq_table->eq[i];
 		eq->hr_dev = hr_dev;
 		eq->eqn = i;
 		eq->irq = hr_dev->irq[i];
 		eq->log_page_size = PAGE_SHIFT;

 		if (i < hr_dev->caps.num_comp_vectors) {
 			/* CEQ */
 			eq_table->eqc_base[i] = hr_dev->reg_base +
 						ROCEE_CAEP_CEQC_SHIFT_0_REG +
 						HNS_ROCE_CEQC_REG_OFFSET * i;
 			eq->type_flag = HNS_ROCE_CEQ;
 			eq->doorbell = hr_dev->reg_base +
 				       ROCEE_CAEP_CEQC_CONS_IDX_0_REG +
 				       HNS_ROCE_CEQC_REG_OFFSET * i;
 			eq->entries = hr_dev->caps.ceqe_depth[i];
 			eq->log_entries = ilog2(eq->entries);
 			eq->eqe_size = sizeof(struct hns_roce_ceqe);
 		} else {
 			/* AEQ */
 			eq_table->eqc_base[i] = hr_dev->reg_base +
 						ROCEE_CAEP_AEQC_AEQE_SHIFT_REG;
 			eq->type_flag = HNS_ROCE_AEQ;
 			eq->doorbell = hr_dev->reg_base +
 				       ROCEE_CAEP_AEQE_CONS_IDX_REG;
 			eq->entries = hr_dev->caps.aeqe_depth;
 			eq->log_entries = ilog2(eq->entries);
 			eq->eqe_size = sizeof(struct hns_roce_aeqe);
 		}
 	}

 	/* Disable irq */
 	hns_roce_int_mask_en(hr_dev);

 	/* Configure CE irq interval and burst num */
 	hns_roce_ce_int_default_cfg(hr_dev);

 	for (i = 0; i < eq_num; i++) {
 		ret = hns_roce_create_eq(hr_dev, &eq_table->eq[i]);
 		if (ret) {
 			dev_err(dev, "eq create failed\n");
 			goto err_create_eq_fail;
 		}
 	}

 	for (j = 0; j < eq_num; j++) {
 		ret = request_irq(eq_table->eq[j].irq, hns_roce_msi_x_interrupt,
 				  0, hr_dev->irq_names[j], eq_table->eq + j);
 		if (ret) {
 			dev_err(dev, "request irq error!\n");
 			goto err_request_irq_fail;
 		}
 	}

 	for (i = 0; i < eq_num; i++)
 		hns_roce_enable_eq(hr_dev, i, EQ_ENABLE);

 	return 0;

 err_request_irq_fail:
 	for (j = j - 1; j >= 0; j--)
 		free_irq(eq_table->eq[j].irq, eq_table->eq + j);

 err_create_eq_fail:
 	for (i = i - 1; i >= 0; i--)
 		hns_roce_free_eq(hr_dev, &eq_table->eq[i]);

 	kfree(eq_table->eqc_base);

 err_eqc_base_alloc_fail:
 	kfree(eq_table->eq);

 	return ret;
 }

 void hns_roce_cleanup_eq_table(struct hns_roce_dev *hr_dev)
 {
 	int i;
 	int eq_num;
 	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;

 	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
 	for (i = 0; i < eq_num; i++) {
 		/* Disable EQ */
 		hns_roce_enable_eq(hr_dev, i, EQ_DISABLE);

 		free_irq(eq_table->eq[i].irq, eq_table->eq + i);

 		hns_roce_free_eq(hr_dev, &eq_table->eq[i]);
 	}

 	kfree(eq_table->eqc_base);
 	kfree(eq_table->eq);
 }
	/*
	* Copyright (c) 2016 Hisilicon Limited.
	*
	* This software is available to you under a choice of one of two
	* licenses. You may choose to be licensed under the terms of the GNU
	* General Public License (GPL) Version 2, available from the file
	* COPYING in the main directory of this source tree, or the
	* OpenIB.org BSD license below:
	*
	* Redistribution and use in source and binary forms, with or
	* without modification, are permitted provided that the following
	* conditions are met:
	*
	* - Redistributions of source code must retain the above
	* copyright notice, this list of conditions and the following
	* disclaimer.
	*
	* - Redistributions in binary form must reproduce the above
	* copyright notice, this list of conditions and the following
	* disclaimer in the documentation and/or other materials
	* provided with the distribution.
	*
	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
	* NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
	* BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
	* ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
	* CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
	* SOFTWARE.
	*/

	#include <linux/platform_device.h>
	#include "hns_roce_common.h"
	#include "hns_roce_device.h"
	#include "hns_roce_eq.h"

	static void eq_set_cons_index(struct hns_roce_eq *eq, int req_not)
	{
	roce_raw_write((eq->cons_index & CONS_INDEX_MASK) \|
	(req_not << eq->log_entries), eq->doorbell);
	/* Memory barrier */
	mb();
	}

	static struct hns_roce_aeqe get_aeqe(struct hns_roce_eq eq, u32 entry)
	{
	unsigned long off = (entry & (eq->entries - 1)) *
	HNS_ROCE_AEQ_ENTRY_SIZE;

	return (struct hns_roce_aeqe )((u8 )
	(eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) +
	off % HNS_ROCE_BA_SIZE);
	}

	static struct hns_roce_aeqe next_aeqe_sw(struct hns_roce_eq eq)
	{
	struct hns_roce_aeqe *aeqe = get_aeqe(eq, eq->cons_index);

	return (roce_get_bit(aeqe->asyn, HNS_ROCE_AEQE_U32_4_OWNER_S) ^
	!!(eq->cons_index & eq->entries)) ? aeqe : NULL;
	}

	static void hns_roce_wq_catas_err_handle(struct hns_roce_dev *hr_dev,
	struct hns_roce_aeqe *aeqe, int qpn)
	{
	struct device *dev = &hr_dev->pdev->dev;

	dev_warn(dev, "Local Work Queue Catastrophic Error.\n");
	switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
	HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
	case HNS_ROCE_LWQCE_QPC_ERROR:
	dev_warn(dev, "QP %d, QPC error.\n", qpn);
	break;
	case HNS_ROCE_LWQCE_MTU_ERROR:
	dev_warn(dev, "QP %d, MTU error.\n", qpn);
	break;
	case HNS_ROCE_LWQCE_WQE_BA_ADDR_ERROR:
	dev_warn(dev, "QP %d, WQE BA addr error.\n", qpn);
	break;
	case HNS_ROCE_LWQCE_WQE_ADDR_ERROR:
	dev_warn(dev, "QP %d, WQE addr error.\n", qpn);
	break;
	case HNS_ROCE_LWQCE_SQ_WQE_SHIFT_ERROR:
	dev_warn(dev, "QP %d, WQE shift error\n", qpn);
	break;
	case HNS_ROCE_LWQCE_SL_ERROR:
	dev_warn(dev, "QP %d, SL error.\n", qpn);
	break;
	case HNS_ROCE_LWQCE_PORT_ERROR:
	dev_warn(dev, "QP %d, port error.\n", qpn);
	break;
	default:
	break;
	}
	}

	static void hns_roce_local_wq_access_err_handle(struct hns_roce_dev *hr_dev,
	struct hns_roce_aeqe *aeqe,
	int qpn)
	{
	struct device *dev = &hr_dev->pdev->dev;

	dev_warn(dev, "Local Access Violation Work Queue Error.\n");
	switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
	HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
	case HNS_ROCE_LAVWQE_R_KEY_VIOLATION:
	dev_warn(dev, "QP %d, R_key violation.\n", qpn);
	break;
	case HNS_ROCE_LAVWQE_LENGTH_ERROR:
	dev_warn(dev, "QP %d, length error.\n", qpn);
	break;
	case HNS_ROCE_LAVWQE_VA_ERROR:
	dev_warn(dev, "QP %d, VA error.\n", qpn);
	break;
	case HNS_ROCE_LAVWQE_PD_ERROR:
	dev_err(dev, "QP %d, PD error.\n", qpn);
	break;
	case HNS_ROCE_LAVWQE_RW_ACC_ERROR:
	dev_warn(dev, "QP %d, rw acc error.\n", qpn);
	break;
	case HNS_ROCE_LAVWQE_KEY_STATE_ERROR:
	dev_warn(dev, "QP %d, key state error.\n", qpn);
	break;
	case HNS_ROCE_LAVWQE_MR_OPERATION_ERROR:
	dev_warn(dev, "QP %d, MR operation error.\n", qpn);
	break;
	default:
	break;
	}
	}

	static void hns_roce_qp_err_handle(struct hns_roce_dev *hr_dev,
	struct hns_roce_aeqe *aeqe,
	int event_type)
	{
	struct device *dev = &hr_dev->pdev->dev;
	int phy_port;
	int qpn;

	qpn = roce_get_field(aeqe->event.qp_event.qp,
	HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_M,
	HNS_ROCE_AEQE_EVENT_QP_EVENT_QP_QPN_S);
	phy_port = roce_get_field(aeqe->event.qp_event.qp,
	HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_M,
	HNS_ROCE_AEQE_EVENT_QP_EVENT_PORT_NUM_S);
	if (qpn <= 1)
	qpn = HNS_ROCE_MAX_PORTS * qpn + phy_port;

	switch (event_type) {
	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
	dev_warn(dev, "Invalid Req Local Work Queue Error.\n"
	"QP %d, phy_port %d.\n", qpn, phy_port);
	break;
	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
	hns_roce_wq_catas_err_handle(hr_dev, aeqe, qpn);
	break;
	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
	hns_roce_local_wq_access_err_handle(hr_dev, aeqe, qpn);
	break;
	default:
	break;
	}

	hns_roce_qp_event(hr_dev, qpn, event_type);
	}

	static void hns_roce_cq_err_handle(struct hns_roce_dev *hr_dev,
	struct hns_roce_aeqe *aeqe,
	int event_type)
	{
	struct device *dev = &hr_dev->pdev->dev;
	u32 cqn;

	cqn = le32_to_cpu(roce_get_field(aeqe->event.cq_event.cq,
	HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_M,
	HNS_ROCE_AEQE_EVENT_CQ_EVENT_CQ_CQN_S));

	switch (event_type) {
	case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
	dev_warn(dev, "CQ 0x%x access err.\n", cqn);
	break;
	case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
	dev_warn(dev, "CQ 0x%x overflow\n", cqn);
	break;
	case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID:
	dev_warn(dev, "CQ 0x%x ID invalid.\n", cqn);
	break;
	default:
	break;
	}

	hns_roce_cq_event(hr_dev, cqn, event_type);
	}

	static void hns_roce_db_overflow_handle(struct hns_roce_dev *hr_dev,
	struct hns_roce_aeqe *aeqe)
	{
	struct device *dev = &hr_dev->pdev->dev;

	switch (roce_get_field(aeqe->asyn, HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_M,
	HNS_ROCE_AEQE_U32_4_EVENT_SUB_TYPE_S)) {
	case HNS_ROCE_DB_SUBTYPE_SDB_OVF:
	dev_warn(dev, "SDB overflow.\n");
	break;
	case HNS_ROCE_DB_SUBTYPE_SDB_ALM_OVF:
	dev_warn(dev, "SDB almost overflow.\n");
	break;
	case HNS_ROCE_DB_SUBTYPE_SDB_ALM_EMP:
	dev_warn(dev, "SDB almost empty.\n");
	break;
	case HNS_ROCE_DB_SUBTYPE_ODB_OVF:
	dev_warn(dev, "ODB overflow.\n");
	break;
	case HNS_ROCE_DB_SUBTYPE_ODB_ALM_OVF:
	dev_warn(dev, "ODB almost overflow.\n");
	break;
	case HNS_ROCE_DB_SUBTYPE_ODB_ALM_EMP:
	dev_warn(dev, "SDB almost empty.\n");
	break;
	default:
	break;
	}
	}

	static int hns_roce_aeq_int(struct hns_roce_dev hr_dev, struct hns_roce_eq eq)
	{
	struct device *dev = &hr_dev->pdev->dev;
	struct hns_roce_aeqe *aeqe;
	int aeqes_found = 0;
	int event_type;

	while ((aeqe = next_aeqe_sw(eq))) {
	dev_dbg(dev, "aeqe = %p, aeqe->asyn.event_type = 0x%lx\n", aeqe,
	roce_get_field(aeqe->asyn,
	HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
	HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S));
	/* Memory barrier */
	rmb();

	event_type = roce_get_field(aeqe->asyn,
	HNS_ROCE_AEQE_U32_4_EVENT_TYPE_M,
	HNS_ROCE_AEQE_U32_4_EVENT_TYPE_S);
	switch (event_type) {
	case HNS_ROCE_EVENT_TYPE_PATH_MIG:
	dev_warn(dev, "PATH MIG not supported\n");
	break;
	case HNS_ROCE_EVENT_TYPE_COMM_EST:
	dev_warn(dev, "COMMUNICATION established\n");
	break;
	case HNS_ROCE_EVENT_TYPE_SQ_DRAINED:
	dev_warn(dev, "SQ DRAINED not supported\n");
	break;
	case HNS_ROCE_EVENT_TYPE_PATH_MIG_FAILED:
	dev_warn(dev, "PATH MIG failed\n");
	break;
	case HNS_ROCE_EVENT_TYPE_INV_REQ_LOCAL_WQ_ERROR:
	case HNS_ROCE_EVENT_TYPE_WQ_CATAS_ERROR:
	case HNS_ROCE_EVENT_TYPE_LOCAL_WQ_ACCESS_ERROR:
	hns_roce_qp_err_handle(hr_dev, aeqe, event_type);
	break;
	case HNS_ROCE_EVENT_TYPE_SRQ_LIMIT_REACH:
	case HNS_ROCE_EVENT_TYPE_SRQ_CATAS_ERROR:
	case HNS_ROCE_EVENT_TYPE_SRQ_LAST_WQE_REACH:
	dev_warn(dev, "SRQ not support!\n");
	break;
	case HNS_ROCE_EVENT_TYPE_CQ_ACCESS_ERROR:
	case HNS_ROCE_EVENT_TYPE_CQ_OVERFLOW:
	case HNS_ROCE_EVENT_TYPE_CQ_ID_INVALID:
	hns_roce_cq_err_handle(hr_dev, aeqe, event_type);
	break;
	case HNS_ROCE_EVENT_TYPE_PORT_CHANGE:
	dev_warn(dev, "port change.\n");
	break;
	case HNS_ROCE_EVENT_TYPE_MB:
	hns_roce_cmd_event(hr_dev,
	le16_to_cpu(aeqe->event.cmd.token),
	aeqe->event.cmd.status,
	le64_to_cpu(aeqe->event.cmd.out_param
	));
	break;
	case HNS_ROCE_EVENT_TYPE_DB_OVERFLOW:
	hns_roce_db_overflow_handle(hr_dev, aeqe);
	break;
	case HNS_ROCE_EVENT_TYPE_CEQ_OVERFLOW:
	dev_warn(dev, "CEQ 0x%lx overflow.\n",
	roce_get_field(aeqe->event.ce_event.ceqe,
	HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_M,
	HNS_ROCE_AEQE_EVENT_CE_EVENT_CEQE_CEQN_S));
	break;
	default:
	dev_warn(dev, "Unhandled event %d on EQ %d at index %u\n",
	event_type, eq->eqn, eq->cons_index);
	break;
	};

	eq->cons_index++;
	aeqes_found = 1;

	if (eq->cons_index > 2 * hr_dev->caps.aeqe_depth - 1) {
	dev_warn(dev, "cons_index overflow, set back to zero\n"
	);
	eq->cons_index = 0;
	}
	}

	eq_set_cons_index(eq, 0);

	return aeqes_found;
	}

	static struct hns_roce_ceqe get_ceqe(struct hns_roce_eq eq, u32 entry)
	{
	unsigned long off = (entry & (eq->entries - 1)) *
	HNS_ROCE_CEQ_ENTRY_SIZE;

	return (struct hns_roce_ceqe )((u8 )
	(eq->buf_list[off / HNS_ROCE_BA_SIZE].buf) +
	off % HNS_ROCE_BA_SIZE);
	}

	static struct hns_roce_ceqe next_ceqe_sw(struct hns_roce_eq eq)
	{
	struct hns_roce_ceqe *ceqe = get_ceqe(eq, eq->cons_index);

	return (!!(roce_get_bit(ceqe->ceqe.comp,
	HNS_ROCE_CEQE_CEQE_COMP_OWNER_S))) ^
	(!!(eq->cons_index & eq->entries)) ? ceqe : NULL;
	}

	static int hns_roce_ceq_int(struct hns_roce_dev hr_dev, struct hns_roce_eq eq)
	{
	struct hns_roce_ceqe *ceqe;
	int ceqes_found = 0;
	u32 cqn;

	while ((ceqe = next_ceqe_sw(eq))) {
	/* Memory barrier */
	rmb();
	cqn = roce_get_field(ceqe->ceqe.comp,
	HNS_ROCE_CEQE_CEQE_COMP_CQN_M,
	HNS_ROCE_CEQE_CEQE_COMP_CQN_S);
	hns_roce_cq_completion(hr_dev, cqn);

	++eq->cons_index;
	ceqes_found = 1;

	if (eq->cons_index > 2 * hr_dev->caps.ceqe_depth[eq->eqn] - 1) {
	dev_warn(&eq->hr_dev->pdev->dev,
	"cons_index overflow, set back to zero\n");
	eq->cons_index = 0;
	}
	}

	eq_set_cons_index(eq, 0);

	return ceqes_found;
	}

	static int hns_roce_aeq_ovf_int(struct hns_roce_dev *hr_dev,
	struct hns_roce_eq *eq)
	{
	struct device *dev = &eq->hr_dev->pdev->dev;
	int eqovf_found = 0;
	u32 caepaemask_val;
	u32 cealmovf_val;
	u32 caepaest_val;
	u32 aeshift_val;
	u32 ceshift_val;
	u32 cemask_val;
	int i = 0;

	/**
	* AEQ overflow ECC mult bit err CEQ overflow alarm
	* must clear interrupt, mask irq, clear irq, cancel mask operation
	*/
	aeshift_val = roce_read(hr_dev, ROCEE_CAEP_AEQC_AEQE_SHIFT_REG);

	if (roce_get_bit(aeshift_val,
	ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQ_ALM_OVF_INT_ST_S) == 1) {
	dev_warn(dev, "AEQ overflow!\n");

	/* Set mask */
	caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
	roce_set_bit(caepaemask_val,
	ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
	HNS_ROCE_INT_MASK_ENABLE);
	roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val);

	/* Clear int state(INT_WC : write 1 clear) */
	caepaest_val = roce_read(hr_dev, ROCEE_CAEP_AE_ST_REG);
	roce_set_bit(caepaest_val,
	ROCEE_CAEP_AE_ST_CAEP_AEQ_ALM_OVF_S, 1);
	roce_write(hr_dev, ROCEE_CAEP_AE_ST_REG, caepaest_val);

	/* Clear mask */
	caepaemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
	roce_set_bit(caepaemask_val,
	ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
	HNS_ROCE_INT_MASK_DISABLE);
	roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, caepaemask_val);
	}

	/* CEQ almost overflow */
	for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) {
	ceshift_val = roce_read(hr_dev, ROCEE_CAEP_CEQC_SHIFT_0_REG +
	i * CEQ_REG_OFFSET);

	if (roce_get_bit(ceshift_val,
	ROCEE_CAEP_CEQC_SHIFT_CAEP_CEQ_ALM_OVF_INT_ST_S) == 1) {
	dev_warn(dev, "CEQ[%d] almost overflow!\n", i);
	eqovf_found++;

	/* Set mask */
	cemask_val = roce_read(hr_dev,
	ROCEE_CAEP_CE_IRQ_MASK_0_REG +
	i * CEQ_REG_OFFSET);
	roce_set_bit(cemask_val,
	ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S,
	HNS_ROCE_INT_MASK_ENABLE);
	roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
	i * CEQ_REG_OFFSET, cemask_val);

	/* Clear int state(INT_WC : write 1 clear) */
	cealmovf_val = roce_read(hr_dev,
	ROCEE_CAEP_CEQ_ALM_OVF_0_REG +
	i * CEQ_REG_OFFSET);
	roce_set_bit(cealmovf_val,
	ROCEE_CAEP_CEQ_ALM_OVF_CAEP_CEQ_ALM_OVF_S,
	1);
	roce_write(hr_dev, ROCEE_CAEP_CEQ_ALM_OVF_0_REG +
	i * CEQ_REG_OFFSET, cealmovf_val);

	/* Clear mask */
	cemask_val = roce_read(hr_dev,
	ROCEE_CAEP_CE_IRQ_MASK_0_REG +
	i * CEQ_REG_OFFSET);
	roce_set_bit(cemask_val,
	ROCEE_CAEP_CE_IRQ_MASK_CAEP_CEQ_ALM_OVF_MASK_S,
	HNS_ROCE_INT_MASK_DISABLE);
	roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
	i * CEQ_REG_OFFSET, cemask_val);
	}
	}

	/* ECC multi-bit error alarm */
	dev_warn(dev, "ECC UCERR ALARM: 0x%x, 0x%x, 0x%x\n",
	roce_read(hr_dev, ROCEE_ECC_UCERR_ALM0_REG),
	roce_read(hr_dev, ROCEE_ECC_UCERR_ALM1_REG),
	roce_read(hr_dev, ROCEE_ECC_UCERR_ALM2_REG));

	dev_warn(dev, "ECC CERR ALARM: 0x%x, 0x%x, 0x%x\n",
	roce_read(hr_dev, ROCEE_ECC_CERR_ALM0_REG),
	roce_read(hr_dev, ROCEE_ECC_CERR_ALM1_REG),
	roce_read(hr_dev, ROCEE_ECC_CERR_ALM2_REG));

	return eqovf_found;
	}

	static int hns_roce_eq_int(struct hns_roce_dev hr_dev, struct hns_roce_eq eq)
	{
	int eqes_found = 0;

	if (likely(eq->type_flag == HNS_ROCE_CEQ))
	/* CEQ irq routine, CEQ is pulse irq, not clear */
	eqes_found = hns_roce_ceq_int(hr_dev, eq);
	else if (likely(eq->type_flag == HNS_ROCE_AEQ))
	/* AEQ irq routine, AEQ is pulse irq, not clear */
	eqes_found = hns_roce_aeq_int(hr_dev, eq);
	else
	/* AEQ queue overflow irq */
	eqes_found = hns_roce_aeq_ovf_int(hr_dev, eq);

	return eqes_found;
	}

	static irqreturn_t hns_roce_msi_x_interrupt(int irq, void *eq_ptr)
	{
	int int_work = 0;
	struct hns_roce_eq *eq = eq_ptr;
	struct hns_roce_dev *hr_dev = eq->hr_dev;

	int_work = hns_roce_eq_int(hr_dev, eq);

	return IRQ_RETVAL(int_work);
	}

	static void hns_roce_enable_eq(struct hns_roce_dev *hr_dev, int eq_num,
	int enable_flag)
	{
	void __iomem *eqc = hr_dev->eq_table.eqc_base[eq_num];
	u32 val;

	val = readl(eqc);

	if (enable_flag)
	roce_set_field(val,
	ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
	ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
	HNS_ROCE_EQ_STAT_VALID);
	else
	roce_set_field(val,
	ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
	ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
	HNS_ROCE_EQ_STAT_INVALID);
	writel(val, eqc);
	}

	static int hns_roce_create_eq(struct hns_roce_dev *hr_dev,
	struct hns_roce_eq *eq)
	{
	void __iomem *eqc = hr_dev->eq_table.eqc_base[eq->eqn];
	struct device *dev = &hr_dev->pdev->dev;
	dma_addr_t tmp_dma_addr;
	u32 eqconsindx_val = 0;
	u32 eqcuridx_val = 0;
	u32 eqshift_val = 0;
	int num_bas = 0;
	int ret;
	int i;

	num_bas = (PAGE_ALIGN(eq->entries * eq->eqe_size) +
	HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE;

	if ((eq->entries * eq->eqe_size) > HNS_ROCE_BA_SIZE) {
	dev_err(dev, "[error]eq buf %d gt ba size(%d) need bas=%d\n",
	(eq->entries * eq->eqe_size), HNS_ROCE_BA_SIZE,
	num_bas);
	return -EINVAL;
	}

	eq->buf_list = kcalloc(num_bas, sizeof(*eq->buf_list), GFP_KERNEL);
	if (!eq->buf_list)
	return -ENOMEM;

	for (i = 0; i < num_bas; ++i) {
	eq->buf_list[i].buf = dma_alloc_coherent(dev, HNS_ROCE_BA_SIZE,
	&tmp_dma_addr,
	GFP_KERNEL);
	if (!eq->buf_list[i].buf) {
	ret = -ENOMEM;
	goto err_out_free_pages;
	}

	eq->buf_list[i].map = tmp_dma_addr;
	memset(eq->buf_list[i].buf, 0, HNS_ROCE_BA_SIZE);
	}
	eq->cons_index = 0;
	roce_set_field(eqshift_val,
	ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_M,
	ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_STATE_S,
	HNS_ROCE_EQ_STAT_INVALID);
	roce_set_field(eqshift_val,
	ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_M,
	ROCEE_CAEP_AEQC_AEQE_SHIFT_CAEP_AEQC_AEQE_SHIFT_S,
	eq->log_entries);
	writel(eqshift_val, eqc);

	/* Configure eq extended address 12~44bit */
	writel((u32)(eq->buf_list[0].map >> 12), (u8 *)eqc + 4);

	/*
	* Configure eq extended address 45~49 bit.
	* 44 = 32 + 12, When evaluating addr to hardware, shift 12 because of
	* using 4K page, and shift more 32 because of
	* caculating the high 32 bit value evaluated to hardware.
	*/
	roce_set_field(eqcuridx_val, ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_M,
	ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQ_BT_H_S,
	eq->buf_list[0].map >> 44);
	roce_set_field(eqcuridx_val,
	ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_M,
	ROCEE_CAEP_AEQE_CUR_IDX_CAEP_AEQE_CUR_IDX_S, 0);
	writel(eqcuridx_val, (u8 *)eqc + 8);

	/* Configure eq consumer index */
	roce_set_field(eqconsindx_val,
	ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_M,
	ROCEE_CAEP_AEQE_CONS_IDX_CAEP_AEQE_CONS_IDX_S, 0);
	writel(eqconsindx_val, (u8 *)eqc + 0xc);

	return 0;

	err_out_free_pages:
	for (i = i - 1; i >= 0; i--)
	dma_free_coherent(dev, HNS_ROCE_BA_SIZE, eq->buf_list[i].buf,
	eq->buf_list[i].map);

	kfree(eq->buf_list);
	return ret;
	}

	static void hns_roce_free_eq(struct hns_roce_dev *hr_dev,
	struct hns_roce_eq *eq)
	{
	int i = 0;
	int npages = (PAGE_ALIGN(eq->eqe_size * eq->entries) +
	HNS_ROCE_BA_SIZE - 1) / HNS_ROCE_BA_SIZE;

	if (!eq->buf_list)
	return;

	for (i = 0; i < npages; ++i)
	dma_free_coherent(&hr_dev->pdev->dev, HNS_ROCE_BA_SIZE,
	eq->buf_list[i].buf, eq->buf_list[i].map);

	kfree(eq->buf_list);
	}

	static void hns_roce_int_mask_en(struct hns_roce_dev *hr_dev)
	{
	int i = 0;
	u32 aemask_val;
	int masken = 0;

	/* AEQ INT */
	aemask_val = roce_read(hr_dev, ROCEE_CAEP_AE_MASK_REG);
	roce_set_bit(aemask_val, ROCEE_CAEP_AE_MASK_CAEP_AEQ_ALM_OVF_MASK_S,
	masken);
	roce_set_bit(aemask_val, ROCEE_CAEP_AE_MASK_CAEP_AE_IRQ_MASK_S, masken);
	roce_write(hr_dev, ROCEE_CAEP_AE_MASK_REG, aemask_val);

	/* CEQ INT */
	for (i = 0; i < hr_dev->caps.num_comp_vectors; i++) {
	/* IRQ mask */
	roce_write(hr_dev, ROCEE_CAEP_CE_IRQ_MASK_0_REG +
	i * CEQ_REG_OFFSET, masken);
	}
	}

	static void hns_roce_ce_int_default_cfg(struct hns_roce_dev *hr_dev)
	{
	/* Configure ce int interval */
	roce_write(hr_dev, ROCEE_CAEP_CE_INTERVAL_CFG_REG,
	HNS_ROCE_CEQ_DEFAULT_INTERVAL);

	/* Configure ce int burst num */
	roce_write(hr_dev, ROCEE_CAEP_CE_BURST_NUM_CFG_REG,
	HNS_ROCE_CEQ_DEFAULT_BURST_NUM);
	}

	int hns_roce_init_eq_table(struct hns_roce_dev *hr_dev)
	{
	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;
	struct device *dev = &hr_dev->pdev->dev;
	struct hns_roce_eq *eq = NULL;
	int eq_num = 0;
	int ret = 0;
	int i = 0;
	int j = 0;

	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
	eq_table->eq = kcalloc(eq_num, sizeof(*eq_table->eq), GFP_KERNEL);
	if (!eq_table->eq)
	return -ENOMEM;

	eq_table->eqc_base = kcalloc(eq_num, sizeof(*eq_table->eqc_base),
	GFP_KERNEL);
	if (!eq_table->eqc_base) {
	ret = -ENOMEM;
	goto err_eqc_base_alloc_fail;
	}

	for (i = 0; i < eq_num; i++) {
	eq = &eq_table->eq[i];
	eq->hr_dev = hr_dev;
	eq->eqn = i;
	eq->irq = hr_dev->irq[i];
	eq->log_page_size = PAGE_SHIFT;

	if (i < hr_dev->caps.num_comp_vectors) {
	/* CEQ */
	eq_table->eqc_base[i] = hr_dev->reg_base +
	ROCEE_CAEP_CEQC_SHIFT_0_REG +
	HNS_ROCE_CEQC_REG_OFFSET * i;
	eq->type_flag = HNS_ROCE_CEQ;
	eq->doorbell = hr_dev->reg_base +
	ROCEE_CAEP_CEQC_CONS_IDX_0_REG +
	HNS_ROCE_CEQC_REG_OFFSET * i;
	eq->entries = hr_dev->caps.ceqe_depth[i];
	eq->log_entries = ilog2(eq->entries);
	eq->eqe_size = sizeof(struct hns_roce_ceqe);
	} else {
	/* AEQ */
	eq_table->eqc_base[i] = hr_dev->reg_base +
	ROCEE_CAEP_AEQC_AEQE_SHIFT_REG;
	eq->type_flag = HNS_ROCE_AEQ;
	eq->doorbell = hr_dev->reg_base +
	ROCEE_CAEP_AEQE_CONS_IDX_REG;
	eq->entries = hr_dev->caps.aeqe_depth;
	eq->log_entries = ilog2(eq->entries);
	eq->eqe_size = sizeof(struct hns_roce_aeqe);
	}
	}

	/* Disable irq */
	hns_roce_int_mask_en(hr_dev);

	/* Configure CE irq interval and burst num */
	hns_roce_ce_int_default_cfg(hr_dev);

	for (i = 0; i < eq_num; i++) {
	ret = hns_roce_create_eq(hr_dev, &eq_table->eq[i]);
	if (ret) {
	dev_err(dev, "eq create failed\n");
	goto err_create_eq_fail;
	}
	}

	for (j = 0; j < eq_num; j++) {
	ret = request_irq(eq_table->eq[j].irq, hns_roce_msi_x_interrupt,
	0, hr_dev->irq_names[j], eq_table->eq + j);
	if (ret) {
	dev_err(dev, "request irq error!\n");
	goto err_request_irq_fail;
	}
	}

	for (i = 0; i < eq_num; i++)
	hns_roce_enable_eq(hr_dev, i, EQ_ENABLE);

	return 0;

	err_request_irq_fail:
	for (j = j - 1; j >= 0; j--)
	free_irq(eq_table->eq[j].irq, eq_table->eq + j);

	err_create_eq_fail:
	for (i = i - 1; i >= 0; i--)
	hns_roce_free_eq(hr_dev, &eq_table->eq[i]);

	kfree(eq_table->eqc_base);

	err_eqc_base_alloc_fail:
	kfree(eq_table->eq);

	return ret;
	}

	void hns_roce_cleanup_eq_table(struct hns_roce_dev *hr_dev)
	{
	int i;
	int eq_num;
	struct hns_roce_eq_table *eq_table = &hr_dev->eq_table;

	eq_num = hr_dev->caps.num_comp_vectors + hr_dev->caps.num_aeq_vectors;
	for (i = 0; i < eq_num; i++) {
	/* Disable EQ */
	hns_roce_enable_eq(hr_dev, i, EQ_DISABLE);

	free_irq(eq_table->eq[i].irq, eq_table->eq + i);

	hns_roce_free_eq(hr_dev, &eq_table->eq[i]);
	}

	kfree(eq_table->eqc_base);
	kfree(eq_table->eq);
	}