/*
 * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
 *
 * Copyright (c) 2011, 2012, Intel Corporation.
 *
 *   Author: Zach Brown <zab@zabbo.net>
 *   Author: Peter J. Braam <braam@clusterfs.com>
 *   Author: Phil Schwan <phil@clusterfs.com>
 *   Author: Eric Barton <eric@bartonsoftware.com>
 *
 *   This file is part of Portals, http://www.sf.net/projects/sandiaportals/
 *
 *   Portals is free software; you can redistribute it and/or
 *   modify it under the terms of version 2 of the GNU General Public
 *   License as published by the Free Software Foundation.
 *
 *   Portals is distributed in the hope that it will be useful,
 *   but WITHOUT ANY WARRANTY; without even the implied warranty of
 *   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 *   GNU General Public License for more details.
 *
 */

#include "socklnd.h"

struct ksock_tx *
ksocknal_alloc_tx(int type, int size)
{
	struct ksock_tx *tx = NULL;

	if (type == KSOCK_MSG_NOOP) {
		LASSERT(size == KSOCK_NOOP_TX_SIZE);

		/* searching for a noop tx in free list */
		spin_lock(&ksocknal_data.ksnd_tx_lock);

		if (!list_empty(&ksocknal_data.ksnd_idle_noop_txs)) {
			tx = list_entry(ksocknal_data.ksnd_idle_noop_txs.next,
					struct ksock_tx, tx_list);
			LASSERT(tx->tx_desc_size == size);
			list_del(&tx->tx_list);
		}

		spin_unlock(&ksocknal_data.ksnd_tx_lock);
	}

	if (!tx)
		LIBCFS_ALLOC(tx, size);

	if (!tx)
		return NULL;

	atomic_set(&tx->tx_refcount, 1);
	tx->tx_zc_aborted = 0;
	tx->tx_zc_capable = 0;
	tx->tx_zc_checked = 0;
	tx->tx_desc_size  = size;

	atomic_inc(&ksocknal_data.ksnd_nactive_txs);

	return tx;
}

struct ksock_tx *
ksocknal_alloc_tx_noop(__u64 cookie, int nonblk)
{
	struct ksock_tx *tx;

	tx = ksocknal_alloc_tx(KSOCK_MSG_NOOP, KSOCK_NOOP_TX_SIZE);
	if (!tx) {
		CERROR("Can't allocate noop tx desc\n");
		return NULL;
	}

	tx->tx_conn    = NULL;
	tx->tx_lnetmsg = NULL;
	tx->tx_kiov    = NULL;
	tx->tx_nkiov   = 0;
	tx->tx_iov     = tx->tx_frags.virt.iov;
	tx->tx_niov    = 1;
	tx->tx_nonblk  = nonblk;

	tx->tx_msg.ksm_csum = 0;
	tx->tx_msg.ksm_type = KSOCK_MSG_NOOP;
	tx->tx_msg.ksm_zc_cookies[0] = 0;
	tx->tx_msg.ksm_zc_cookies[1] = cookie;

	return tx;
}

void
ksocknal_free_tx(struct ksock_tx *tx)
{
	atomic_dec(&ksocknal_data.ksnd_nactive_txs);

	if (!tx->tx_lnetmsg && tx->tx_desc_size == KSOCK_NOOP_TX_SIZE) {
		/* it's a noop tx */
		spin_lock(&ksocknal_data.ksnd_tx_lock);

		list_add(&tx->tx_list, &ksocknal_data.ksnd_idle_noop_txs);

		spin_unlock(&ksocknal_data.ksnd_tx_lock);
	} else {
		LIBCFS_FREE(tx, tx->tx_desc_size);
	}
}

static int
ksocknal_send_iov(struct ksock_conn *conn, struct ksock_tx *tx)
{
	struct kvec *iov = tx->tx_iov;
	int nob;
	int rc;

	LASSERT(tx->tx_niov > 0);

	/* Never touch tx->tx_iov inside ksocknal_lib_send_iov() */
	rc = ksocknal_lib_send_iov(conn, tx);

	if (rc <= 0)			    /* sent nothing? */
		return rc;

	nob = rc;
	LASSERT(nob <= tx->tx_resid);
	tx->tx_resid -= nob;

	/* "consume" iov */
	do {
		LASSERT(tx->tx_niov > 0);

		if (nob < (int)iov->iov_len) {
			iov->iov_base = (void *)((char *)iov->iov_base + nob);
			iov->iov_len -= nob;
			return rc;
		}

		nob -= iov->iov_len;
		tx->tx_iov = ++iov;
		tx->tx_niov--;
	} while (nob);

	return rc;
}

static int
ksocknal_send_kiov(struct ksock_conn *conn, struct ksock_tx *tx)
{
	struct bio_vec *kiov = tx->tx_kiov;
	int nob;
	int rc;

	LASSERT(!tx->tx_niov);
	LASSERT(tx->tx_nkiov > 0);

	/* Never touch tx->tx_kiov inside ksocknal_lib_send_kiov() */
	rc = ksocknal_lib_send_kiov(conn, tx);

	if (rc <= 0)			    /* sent nothing? */
		return rc;

	nob = rc;
	LASSERT(nob <= tx->tx_resid);
	tx->tx_resid -= nob;

	/* "consume" kiov */
	do {
		LASSERT(tx->tx_nkiov > 0);

		if (nob < (int)kiov->bv_len) {
			kiov->bv_offset += nob;
			kiov->bv_len -= nob;
			return rc;
		}

		nob -= (int)kiov->bv_len;
		tx->tx_kiov = ++kiov;
		tx->tx_nkiov--;
	} while (nob);

	return rc;
}

static int
ksocknal_transmit(struct ksock_conn *conn, struct ksock_tx *tx)
{
	int rc;
	int bufnob;

	if (ksocknal_data.ksnd_stall_tx) {
		set_current_state(TASK_UNINTERRUPTIBLE);
		schedule_timeout(cfs_time_seconds(ksocknal_data.ksnd_stall_tx));
	}

	LASSERT(tx->tx_resid);

	rc = ksocknal_connsock_addref(conn);
	if (rc) {
		LASSERT(conn->ksnc_closing);
		return -ESHUTDOWN;
	}

	do {
		if (ksocknal_data.ksnd_enomem_tx > 0) {
			/* testing... */
			ksocknal_data.ksnd_enomem_tx--;
			rc = -EAGAIN;
		} else if (tx->tx_niov) {
			rc = ksocknal_send_iov(conn, tx);
		} else {
			rc = ksocknal_send_kiov(conn, tx);
		}

		bufnob = conn->ksnc_sock->sk->sk_wmem_queued;
		if (rc > 0)		     /* sent something? */
			conn->ksnc_tx_bufnob += rc; /* account it */

		if (bufnob < conn->ksnc_tx_bufnob) {
			/*
			 * allocated send buffer bytes < computed; infer
			 * something got ACKed
			 */
			conn->ksnc_tx_deadline =
				cfs_time_shift(*ksocknal_tunables.ksnd_timeout);
			conn->ksnc_peer->ksnp_last_alive = cfs_time_current();
			conn->ksnc_tx_bufnob = bufnob;
			mb();
		}

		if (rc <= 0) { /* Didn't write anything? */

			if (!rc) /* some stacks return 0 instead of -EAGAIN */
				rc = -EAGAIN;

			/* Check if EAGAIN is due to memory pressure */
			if (rc == -EAGAIN && ksocknal_lib_memory_pressure(conn))
				rc = -ENOMEM;

			break;
		}

		/* socket's wmem_queued now includes 'rc' bytes */
		atomic_sub(rc, &conn->ksnc_tx_nob);
		rc = 0;

	} while (tx->tx_resid);

	ksocknal_connsock_decref(conn);
	return rc;
}

static int
ksocknal_recv_iov(struct ksock_conn *conn)
{
	struct kvec *iov = conn->ksnc_rx_iov;
	int nob;
	int rc;

	LASSERT(conn->ksnc_rx_niov > 0);

	/*
	 * Never touch conn->ksnc_rx_iov or change connection
	 * status inside ksocknal_lib_recv_iov
	 */
	rc = ksocknal_lib_recv_iov(conn);

	if (rc <= 0)
		return rc;

	/* received something... */
	nob = rc;

	conn->ksnc_peer->ksnp_last_alive = cfs_time_current();
	conn->ksnc_rx_deadline =
		cfs_time_shift(*ksocknal_tunables.ksnd_timeout);
	mb();		       /* order with setting rx_started */
	conn->ksnc_rx_started = 1;

	conn->ksnc_rx_nob_wanted -= nob;
	conn->ksnc_rx_nob_left -= nob;

	do {
		LASSERT(conn->ksnc_rx_niov > 0);

		if (nob < (int)iov->iov_len) {
			iov->iov_len -= nob;
			iov->iov_base += nob;
			return -EAGAIN;
		}

		nob -= iov->iov_len;
		conn->ksnc_rx_iov = ++iov;
		conn->ksnc_rx_niov--;
	} while (nob);

	return rc;
}

static int
ksocknal_recv_kiov(struct ksock_conn *conn)
{
	struct bio_vec *kiov = conn->ksnc_rx_kiov;
	int nob;
	int rc;

	LASSERT(conn->ksnc_rx_nkiov > 0);

	/*
	 * Never touch conn->ksnc_rx_kiov or change connection
	 * status inside ksocknal_lib_recv_iov
	 */
	rc = ksocknal_lib_recv_kiov(conn);

	if (rc <= 0)
		return rc;

	/* received something... */
	nob = rc;

	conn->ksnc_peer->ksnp_last_alive = cfs_time_current();
	conn->ksnc_rx_deadline =
		cfs_time_shift(*ksocknal_tunables.ksnd_timeout);
	mb();		       /* order with setting rx_started */
	conn->ksnc_rx_started = 1;

	conn->ksnc_rx_nob_wanted -= nob;
	conn->ksnc_rx_nob_left -= nob;

	do {
		LASSERT(conn->ksnc_rx_nkiov > 0);

		if (nob < (int)kiov->bv_len) {
			kiov->bv_offset += nob;
			kiov->bv_len -= nob;
			return -EAGAIN;
		}

		nob -= kiov->bv_len;
		conn->ksnc_rx_kiov = ++kiov;
		conn->ksnc_rx_nkiov--;
	} while (nob);

	return 1;
}

static int
ksocknal_receive(struct ksock_conn *conn)
{
	/*
	 * Return 1 on success, 0 on EOF, < 0 on error.
	 * Caller checks ksnc_rx_nob_wanted to determine
	 * progress/completion.
	 */
	int rc;

	if (ksocknal_data.ksnd_stall_rx) {
		set_current_state(TASK_UNINTERRUPTIBLE);
		schedule_timeout(cfs_time_seconds(ksocknal_data.ksnd_stall_rx));
	}

	rc = ksocknal_connsock_addref(conn);
	if (rc) {
		LASSERT(conn->ksnc_closing);
		return -ESHUTDOWN;
	}

	for (;;) {
		if (conn->ksnc_rx_niov)
			rc = ksocknal_recv_iov(conn);
		else
			rc = ksocknal_recv_kiov(conn);

		if (rc <= 0) {
			/* error/EOF or partial receive */
			if (rc == -EAGAIN) {
				rc = 1;
			} else if (!rc && conn->ksnc_rx_started) {
				/* EOF in the middle of a message */
				rc = -EPROTO;
			}
			break;
		}

		/* Completed a fragment */

		if (!conn->ksnc_rx_nob_wanted) {
			rc = 1;
			break;
		}
	}

	ksocknal_connsock_decref(conn);
	return rc;
}

void
ksocknal_tx_done(struct lnet_ni *ni, struct ksock_tx *tx)
{
	struct lnet_msg *lnetmsg = tx->tx_lnetmsg;
	int rc = (!tx->tx_resid && !tx->tx_zc_aborted) ? 0 : -EIO;

	LASSERT(ni || tx->tx_conn);

	if (tx->tx_conn)
		ksocknal_conn_decref(tx->tx_conn);

	if (!ni && tx->tx_conn)
		ni = tx->tx_conn->ksnc_peer->ksnp_ni;

	ksocknal_free_tx(tx);
	if (lnetmsg) /* KSOCK_MSG_NOOP go without lnetmsg */
		lnet_finalize(ni, lnetmsg, rc);
}

void
ksocknal_txlist_done(struct lnet_ni *ni, struct list_head *txlist, int error)
{
	struct ksock_tx *tx;

	while (!list_empty(txlist)) {
		tx = list_entry(txlist->next, struct ksock_tx, tx_list);

		if (error && tx->tx_lnetmsg) {
			CNETERR("Deleting packet type %d len %d %s->%s\n",
				le32_to_cpu(tx->tx_lnetmsg->msg_hdr.type),
				le32_to_cpu(tx->tx_lnetmsg->msg_hdr.payload_length),
				libcfs_nid2str(le64_to_cpu(tx->tx_lnetmsg->msg_hdr.src_nid)),
				libcfs_nid2str(le64_to_cpu(tx->tx_lnetmsg->msg_hdr.dest_nid)));
		} else if (error) {
			CNETERR("Deleting noop packet\n");
		}

		list_del(&tx->tx_list);

		LASSERT(atomic_read(&tx->tx_refcount) == 1);
		ksocknal_tx_done(ni, tx);
	}
}

static void
ksocknal_check_zc_req(struct ksock_tx *tx)
{
	struct ksock_conn *conn = tx->tx_conn;
	struct ksock_peer *peer = conn->ksnc_peer;

	/*
	 * Set tx_msg.ksm_zc_cookies[0] to a unique non-zero cookie and add tx
	 * to ksnp_zc_req_list if some fragment of this message should be sent
	 * zero-copy.  Our peer will send an ACK containing this cookie when
	 * she has received this message to tell us we can signal completion.
	 * tx_msg.ksm_zc_cookies[0] remains non-zero while tx is on
	 * ksnp_zc_req_list.
	 */
	LASSERT(tx->tx_msg.ksm_type != KSOCK_MSG_NOOP);
	LASSERT(tx->tx_zc_capable);

	tx->tx_zc_checked = 1;

	if (conn->ksnc_proto == &ksocknal_protocol_v1x ||
	    !conn->ksnc_zc_capable)
		return;

	/*
	 * assign cookie and queue tx to pending list, it will be released when
	 * a matching ack is received. See ksocknal_handle_zcack()
	 */
	ksocknal_tx_addref(tx);

	spin_lock(&peer->ksnp_lock);

	/* ZC_REQ is going to be pinned to the peer */
	tx->tx_deadline =
		cfs_time_shift(*ksocknal_tunables.ksnd_timeout);

	LASSERT(!tx->tx_msg.ksm_zc_cookies[0]);

	tx->tx_msg.ksm_zc_cookies[0] = peer->ksnp_zc_next_cookie++;

	if (!peer->ksnp_zc_next_cookie)
		peer->ksnp_zc_next_cookie = SOCKNAL_KEEPALIVE_PING + 1;

	list_add_tail(&tx->tx_zc_list, &peer->ksnp_zc_req_list);

	spin_unlock(&peer->ksnp_lock);
}

static void
ksocknal_uncheck_zc_req(struct ksock_tx *tx)
{
	struct ksock_peer *peer = tx->tx_conn->ksnc_peer;

	LASSERT(tx->tx_msg.ksm_type != KSOCK_MSG_NOOP);
	LASSERT(tx->tx_zc_capable);

	tx->tx_zc_checked = 0;

	spin_lock(&peer->ksnp_lock);

	if (!tx->tx_msg.ksm_zc_cookies[0]) {
		/* Not waiting for an ACK */
		spin_unlock(&peer->ksnp_lock);
		return;
	}

	tx->tx_msg.ksm_zc_cookies[0] = 0;
	list_del(&tx->tx_zc_list);

	spin_unlock(&peer->ksnp_lock);

	ksocknal_tx_decref(tx);
}

static int
ksocknal_process_transmit(struct ksock_conn *conn, struct ksock_tx *tx)
{
	int rc;

	if (tx->tx_zc_capable && !tx->tx_zc_checked)
		ksocknal_check_zc_req(tx);

	rc = ksocknal_transmit(conn, tx);

	CDEBUG(D_NET, "send(%d) %d\n", tx->tx_resid, rc);

	if (!tx->tx_resid) {
		/* Sent everything OK */
		LASSERT(!rc);

		return 0;
	}

	if (rc == -EAGAIN)
		return rc;

	if (rc == -ENOMEM) {
		static int counter;

		counter++;   /* exponential backoff warnings */
		if ((counter & (-counter)) == counter)
			CWARN("%u ENOMEM tx %p\n", counter, conn);

		/* Queue on ksnd_enomem_conns for retry after a timeout */
		spin_lock_bh(&ksocknal_data.ksnd_reaper_lock);

		/* enomem list takes over scheduler's ref... */
		LASSERT(conn->ksnc_tx_scheduled);
		list_add_tail(&conn->ksnc_tx_list,
			      &ksocknal_data.ksnd_enomem_conns);
		if (!cfs_time_aftereq(cfs_time_add(cfs_time_current(),
						   SOCKNAL_ENOMEM_RETRY),
				   ksocknal_data.ksnd_reaper_waketime))
			wake_up(&ksocknal_data.ksnd_reaper_waitq);

		spin_unlock_bh(&ksocknal_data.ksnd_reaper_lock);
		return rc;
	}

	/* Actual error */
	LASSERT(rc < 0);

	if (!conn->ksnc_closing) {
		switch (rc) {
		case -ECONNRESET:
			LCONSOLE_WARN("Host %pI4h reset our connection while we were sending data; it may have rebooted.\n",
				      &conn->ksnc_ipaddr);
			break;
		default:
			LCONSOLE_WARN("There was an unexpected network error while writing to %pI4h: %d.\n",
				      &conn->ksnc_ipaddr, rc);
			break;
		}
		CDEBUG(D_NET, "[%p] Error %d on write to %s ip %pI4h:%d\n",
		       conn, rc,
		       libcfs_id2str(conn->ksnc_peer->ksnp_id),
		       &conn->ksnc_ipaddr,
		       conn->ksnc_port);
	}

	if (tx->tx_zc_checked)
		ksocknal_uncheck_zc_req(tx);

	/* it's not an error if conn is being closed */
	ksocknal_close_conn_and_siblings(conn, (conn->ksnc_closing) ? 0 : rc);

	return rc;
}

static void
ksocknal_launch_connection_locked(struct ksock_route *route)
{
	/* called holding write lock on ksnd_global_lock */

	LASSERT(!route->ksnr_scheduled);
	LASSERT(!route->ksnr_connecting);
	LASSERT(ksocknal_route_mask() & ~route->ksnr_connected);

	route->ksnr_scheduled = 1;	      /* scheduling conn for connd */
	ksocknal_route_addref(route);	   /* extra ref for connd */

	spin_lock_bh(&ksocknal_data.ksnd_connd_lock);

	list_add_tail(&route->ksnr_connd_list,
		      &ksocknal_data.ksnd_connd_routes);
	wake_up(&ksocknal_data.ksnd_connd_waitq);

	spin_unlock_bh(&ksocknal_data.ksnd_connd_lock);
}

void
ksocknal_launch_all_connections_locked(struct ksock_peer *peer)
{
	struct ksock_route *route;

	/* called holding write lock on ksnd_global_lock */
	for (;;) {
		/* launch any/all connections that need it */
		route = ksocknal_find_connectable_route_locked(peer);
		if (!route)
			return;

		ksocknal_launch_connection_locked(route);
	}
}

struct ksock_conn *
ksocknal_find_conn_locked(struct ksock_peer *peer, struct ksock_tx *tx,
			  int nonblk)
{
	struct list_head *tmp;
	struct ksock_conn *conn;
	struct ksock_conn *typed = NULL;
	struct ksock_conn *fallback = NULL;
	int tnob = 0;
	int fnob = 0;

	list_for_each(tmp, &peer->ksnp_conns) {
		struct ksock_conn *c;
		int nob, rc;

		c = list_entry(tmp, struct ksock_conn, ksnc_list);
		nob = atomic_read(&c->ksnc_tx_nob) +
		      c->ksnc_sock->sk->sk_wmem_queued;

		LASSERT(!c->ksnc_closing);
		LASSERT(c->ksnc_proto &&
			c->ksnc_proto->pro_match_tx);

		rc = c->ksnc_proto->pro_match_tx(c, tx, nonblk);

		switch (rc) {
		default:
			LBUG();
		case SOCKNAL_MATCH_NO: /* protocol rejected the tx */
			continue;

		case SOCKNAL_MATCH_YES: /* typed connection */
			if (!typed || tnob > nob ||
			    (tnob == nob && *ksocknal_tunables.ksnd_round_robin &&
			     cfs_time_after(typed->ksnc_tx_last_post, c->ksnc_tx_last_post))) {
				typed = c;
				tnob  = nob;
			}
			break;

		case SOCKNAL_MATCH_MAY: /* fallback connection */
			if (!fallback || fnob > nob ||
			    (fnob == nob && *ksocknal_tunables.ksnd_round_robin &&
			     cfs_time_after(fallback->ksnc_tx_last_post, c->ksnc_tx_last_post))) {
				fallback = c;
				fnob = nob;
			}
			break;
		}
	}

	/* prefer the typed selection */
	conn = (typed) ? typed : fallback;

	if (conn)
		conn->ksnc_tx_last_post = cfs_time_current();

	return conn;
}

void
ksocknal_tx_prep(struct ksock_conn *conn, struct ksock_tx *tx)
{
	conn->ksnc_proto->pro_pack(tx);

	atomic_add(tx->tx_nob, &conn->ksnc_tx_nob);
	ksocknal_conn_addref(conn); /* +1 ref for tx */
	tx->tx_conn = conn;
}

void
ksocknal_queue_tx_locked(struct ksock_tx *tx, struct ksock_conn *conn)
{
	struct ksock_sched *sched = conn->ksnc_scheduler;
	struct ksock_msg *msg = &tx->tx_msg;
	struct ksock_tx *ztx = NULL;
	int bufnob = 0;

	/*
	 * called holding global lock (read or irq-write) and caller may
	 * not have dropped this lock between finding conn and calling me,
	 * so we don't need the {get,put}connsock dance to deref
	 * ksnc_sock...
	 */
	LASSERT(!conn->ksnc_closing);

	CDEBUG(D_NET, "Sending to %s ip %pI4h:%d\n",
	       libcfs_id2str(conn->ksnc_peer->ksnp_id),
	       &conn->ksnc_ipaddr, conn->ksnc_port);

	ksocknal_tx_prep(conn, tx);

	/*
	 * Ensure the frags we've been given EXACTLY match the number of
	 * bytes we want to send.  Many TCP/IP stacks disregard any total
	 * size parameters passed to them and just look at the frags.
	 *
	 * We always expect at least 1 mapped fragment containing the
	 * complete ksocknal message header.
	 */
	LASSERT(lnet_iov_nob(tx->tx_niov, tx->tx_iov) +
		lnet_kiov_nob(tx->tx_nkiov, tx->tx_kiov) ==
		(unsigned int)tx->tx_nob);
	LASSERT(tx->tx_niov >= 1);
	LASSERT(tx->tx_resid == tx->tx_nob);

	CDEBUG(D_NET, "Packet %p type %d, nob %d niov %d nkiov %d\n",
	       tx, (tx->tx_lnetmsg) ? tx->tx_lnetmsg->msg_hdr.type :
					      KSOCK_MSG_NOOP,
	       tx->tx_nob, tx->tx_niov, tx->tx_nkiov);

	/*
	 * FIXME: SOCK_WMEM_QUEUED and SOCK_ERROR could block in __DARWIN8__
	 * but they're used inside spinlocks a lot.
	 */
	bufnob = conn->ksnc_sock->sk->sk_wmem_queued;
	spin_lock_bh(&sched->kss_lock);

	if (list_empty(&conn->ksnc_tx_queue) && !bufnob) {
		/* First packet starts the timeout */
		conn->ksnc_tx_deadline =
			cfs_time_shift(*ksocknal_tunables.ksnd_timeout);
		if (conn->ksnc_tx_bufnob > 0) /* something got ACKed */
			conn->ksnc_peer->ksnp_last_alive = cfs_time_current();
		conn->ksnc_tx_bufnob = 0;
		mb(); /* order with adding to tx_queue */
	}

	if (msg->ksm_type == KSOCK_MSG_NOOP) {
		/*
		 * The packet is noop ZC ACK, try to piggyback the ack_cookie
		 * on a normal packet so I don't need to send it
		 */
		LASSERT(msg->ksm_zc_cookies[1]);
		LASSERT(conn->ksnc_proto->pro_queue_tx_zcack);

		/* ZC ACK piggybacked on ztx release tx later */
		if (conn->ksnc_proto->pro_queue_tx_zcack(conn, tx, 0))
			ztx = tx;
	} else {
		/*
		 * It's a normal packet - can it piggback a noop zc-ack that
		 * has been queued already?
		 */
		LASSERT(!msg->ksm_zc_cookies[1]);
		LASSERT(conn->ksnc_proto->pro_queue_tx_msg);

		ztx = conn->ksnc_proto->pro_queue_tx_msg(conn, tx);
		/* ztx will be released later */
	}

	if (ztx) {
		atomic_sub(ztx->tx_nob, &conn->ksnc_tx_nob);
		list_add_tail(&ztx->tx_list, &sched->kss_zombie_noop_txs);
	}

	if (conn->ksnc_tx_ready &&      /* able to send */
	    !conn->ksnc_tx_scheduled) { /* not scheduled to send */
		/* +1 ref for scheduler */
		ksocknal_conn_addref(conn);
		list_add_tail(&conn->ksnc_tx_list, &sched->kss_tx_conns);
		conn->ksnc_tx_scheduled = 1;
		wake_up(&sched->kss_waitq);
	}

	spin_unlock_bh(&sched->kss_lock);
}

struct ksock_route *
ksocknal_find_connectable_route_locked(struct ksock_peer *peer)
{
	unsigned long now = cfs_time_current();
	struct list_head *tmp;
	struct ksock_route *route;

	list_for_each(tmp, &peer->ksnp_routes) {
		route = list_entry(tmp, struct ksock_route, ksnr_list);

		LASSERT(!route->ksnr_connecting || route->ksnr_scheduled);

		/* connections being established */
		if (route->ksnr_scheduled)
			continue;

		/* all route types connected ? */
		if (!(ksocknal_route_mask() & ~route->ksnr_connected))
			continue;

		if (!(!route->ksnr_retry_interval || /* first attempt */
		      cfs_time_aftereq(now, route->ksnr_timeout))) {
			CDEBUG(D_NET,
			       "Too soon to retry route %pI4h (cnted %d, interval %ld, %ld secs later)\n",
			       &route->ksnr_ipaddr,
			       route->ksnr_connected,
			       route->ksnr_retry_interval,
			       cfs_duration_sec(route->ksnr_timeout - now));
			continue;
		}

		return route;
	}

	return NULL;
}

struct ksock_route *
ksocknal_find_connecting_route_locked(struct ksock_peer *peer)
{
	struct list_head *tmp;
	struct ksock_route *route;

	list_for_each(tmp, &peer->ksnp_routes) {
		route = list_entry(tmp, struct ksock_route, ksnr_list);

		LASSERT(!route->ksnr_connecting || route->ksnr_scheduled);

		if (route->ksnr_scheduled)
			return route;
	}

	return NULL;
}

int
ksocknal_launch_packet(struct lnet_ni *ni, struct ksock_tx *tx,
		       struct lnet_process_id id)
{
	struct ksock_peer *peer;
	struct ksock_conn *conn;
	rwlock_t *g_lock;
	int retry;
	int rc;

	LASSERT(!tx->tx_conn);

	g_lock = &ksocknal_data.ksnd_global_lock;

	for (retry = 0;; retry = 1) {
		read_lock(g_lock);
		peer = ksocknal_find_peer_locked(ni, id);
		if (peer) {
			if (!ksocknal_find_connectable_route_locked(peer)) {
				conn = ksocknal_find_conn_locked(peer, tx, tx->tx_nonblk);
				if (conn) {
					/*
					 * I've got no routes that need to be
					 * connecting and I do have an actual
					 * connection...
					 */
					ksocknal_queue_tx_locked(tx, conn);
					read_unlock(g_lock);
					return 0;
				}
			}
		}

		/* I'll need a write lock... */
		read_unlock(g_lock);

		write_lock_bh(g_lock);

		peer = ksocknal_find_peer_locked(ni, id);
		if (peer)
			break;

		write_unlock_bh(g_lock);

		if (id.pid & LNET_PID_USERFLAG) {
			CERROR("Refusing to create a connection to userspace process %s\n",
			       libcfs_id2str(id));
			return -EHOSTUNREACH;
		}

		if (retry) {
			CERROR("Can't find peer %s\n", libcfs_id2str(id));
			return -EHOSTUNREACH;
		}

		rc = ksocknal_add_peer(ni, id,
				       LNET_NIDADDR(id.nid),
				       lnet_acceptor_port());
		if (rc) {
			CERROR("Can't add peer %s: %d\n",
			       libcfs_id2str(id), rc);
			return rc;
		}
	}

	ksocknal_launch_all_connections_locked(peer);

	conn = ksocknal_find_conn_locked(peer, tx, tx->tx_nonblk);
	if (conn) {
		/* Connection exists; queue message on it */
		ksocknal_queue_tx_locked(tx, conn);
		write_unlock_bh(g_lock);
		return 0;
	}

	if (peer->ksnp_accepting > 0 ||
	    ksocknal_find_connecting_route_locked(peer)) {
		/* the message is going to be pinned to the peer */
		tx->tx_deadline =
			cfs_time_shift(*ksocknal_tunables.ksnd_timeout);

		/* Queue the message until a connection is established */
		list_add_tail(&tx->tx_list, &peer->ksnp_tx_queue);
		write_unlock_bh(g_lock);
		return 0;
	}

	write_unlock_bh(g_lock);

	/* NB Routes may be ignored if connections to them failed recently */
	CNETERR("No usable routes to %s\n", libcfs_id2str(id));
	return -EHOSTUNREACH;
}

int
ksocknal_send(struct lnet_ni *ni, void *private, struct lnet_msg *lntmsg)
{
	int mpflag = 1;
	int type = lntmsg->msg_type;
	struct lnet_process_id target = lntmsg->msg_target;
	unsigned int payload_niov = lntmsg->msg_niov;
	struct kvec *payload_iov = lntmsg->msg_iov;
	struct bio_vec *payload_kiov = lntmsg->msg_kiov;
	unsigned int payload_offset = lntmsg->msg_offset;
	unsigned int payload_nob = lntmsg->msg_len;
	struct ksock_tx *tx;
	int desc_size;
	int rc;

	/*
	 * NB 'private' is different depending on what we're sending.
	 * Just ignore it...
	 */
	CDEBUG(D_NET, "sending %u bytes in %d frags to %s\n",
	       payload_nob, payload_niov, libcfs_id2str(target));

	LASSERT(!payload_nob || payload_niov > 0);
	LASSERT(payload_niov <= LNET_MAX_IOV);
	/* payload is either all vaddrs or all pages */
	LASSERT(!(payload_kiov && payload_iov));
	LASSERT(!in_interrupt());

	if (payload_iov)
		desc_size = offsetof(struct ksock_tx,
				     tx_frags.virt.iov[1 + payload_niov]);
	else
		desc_size = offsetof(struct ksock_tx,
				     tx_frags.paged.kiov[payload_niov]);

	if (lntmsg->msg_vmflush)
		mpflag = cfs_memory_pressure_get_and_set();
	tx = ksocknal_alloc_tx(KSOCK_MSG_LNET, desc_size);
	if (!tx) {
		CERROR("Can't allocate tx desc type %d size %d\n",
		       type, desc_size);
		if (lntmsg->msg_vmflush)
			cfs_memory_pressure_restore(mpflag);
		return -ENOMEM;
	}

	tx->tx_conn = NULL;		     /* set when assigned a conn */
	tx->tx_lnetmsg = lntmsg;

	if (payload_iov) {
		tx->tx_kiov = NULL;
		tx->tx_nkiov = 0;
		tx->tx_iov = tx->tx_frags.virt.iov;
		tx->tx_niov = 1 +
			      lnet_extract_iov(payload_niov, &tx->tx_iov[1],
					       payload_niov, payload_iov,
					       payload_offset, payload_nob);
	} else {
		tx->tx_niov = 1;
		tx->tx_iov = &tx->tx_frags.paged.iov;
		tx->tx_kiov = tx->tx_frags.paged.kiov;
		tx->tx_nkiov = lnet_extract_kiov(payload_niov, tx->tx_kiov,
						 payload_niov, payload_kiov,
						 payload_offset, payload_nob);

		if (payload_nob >= *ksocknal_tunables.ksnd_zc_min_payload)
			tx->tx_zc_capable = 1;
	}

	tx->tx_msg.ksm_csum = 0;
	tx->tx_msg.ksm_type = KSOCK_MSG_LNET;
	tx->tx_msg.ksm_zc_cookies[0] = 0;
	tx->tx_msg.ksm_zc_cookies[1] = 0;

	/* The first fragment will be set later in pro_pack */
	rc = ksocknal_launch_packet(ni, tx, target);
	if (!mpflag)
		cfs_memory_pressure_restore(mpflag);

	if (!rc)
		return 0;

	ksocknal_free_tx(tx);
	return -EIO;
}

int
ksocknal_thread_start(int (*fn)(void *arg), void *arg, char *name)
{
	struct task_struct *task = kthread_run(fn, arg, "%s", name);

	if (IS_ERR(task))
		return PTR_ERR(task);

	write_lock_bh(&ksocknal_data.ksnd_global_lock);
	ksocknal_data.ksnd_nthreads++;
	write_unlock_bh(&ksocknal_data.ksnd_global_lock);
	return 0;
}

void
ksocknal_thread_fini(void)
{
	write_lock_bh(&ksocknal_data.ksnd_global_lock);
	ksocknal_data.ksnd_nthreads--;
	write_unlock_bh(&ksocknal_data.ksnd_global_lock);
}

int
ksocknal_new_packet(struct ksock_conn *conn, int nob_to_skip)
{
	static char ksocknal_slop_buffer[4096];

	int nob;
	unsigned int niov;
	int skipped;

	LASSERT(conn->ksnc_proto);

	if (*ksocknal_tunables.ksnd_eager_ack & conn->ksnc_type) {
		/* Remind the socket to ack eagerly... */
		ksocknal_lib_eager_ack(conn);
	}

	if (!nob_to_skip) {	 /* right at next packet boundary now */
		conn->ksnc_rx_started = 0;
		mb();		       /* racing with timeout thread */

		switch (conn->ksnc_proto->pro_version) {
		case  KSOCK_PROTO_V2:
		case  KSOCK_PROTO_V3:
			conn->ksnc_rx_state = SOCKNAL_RX_KSM_HEADER;
			conn->ksnc_rx_iov = (struct kvec *)&conn->ksnc_rx_iov_space;
			conn->ksnc_rx_iov[0].iov_base = &conn->ksnc_msg;

			conn->ksnc_rx_nob_wanted = offsetof(struct ksock_msg, ksm_u);
			conn->ksnc_rx_nob_left = offsetof(struct ksock_msg, ksm_u);
			conn->ksnc_rx_iov[0].iov_len = offsetof(struct ksock_msg, ksm_u);
			break;

		case KSOCK_PROTO_V1:
			/* Receiving bare struct lnet_hdr */
			conn->ksnc_rx_state = SOCKNAL_RX_LNET_HEADER;
			conn->ksnc_rx_nob_wanted = sizeof(struct lnet_hdr);
			conn->ksnc_rx_nob_left = sizeof(struct lnet_hdr);

			conn->ksnc_rx_iov = (struct kvec *)&conn->ksnc_rx_iov_space;
			conn->ksnc_rx_iov[0].iov_base = &conn->ksnc_msg.ksm_u.lnetmsg;
			conn->ksnc_rx_iov[0].iov_len = sizeof(struct lnet_hdr);
			break;

		default:
			LBUG();
		}
		conn->ksnc_rx_niov = 1;

		conn->ksnc_rx_kiov = NULL;
		conn->ksnc_rx_nkiov = 0;
		conn->ksnc_rx_csum = ~0;
		return 1;
	}

	/*
	 * Set up to skip as much as possible now.  If there's more left
	 * (ran out of iov entries) we'll get called again
	 */
	conn->ksnc_rx_state = SOCKNAL_RX_SLOP;
	conn->ksnc_rx_nob_left = nob_to_skip;
	conn->ksnc_rx_iov = (struct kvec *)&conn->ksnc_rx_iov_space;
	skipped = 0;
	niov = 0;

	do {
		nob = min_t(int, nob_to_skip, sizeof(ksocknal_slop_buffer));

		conn->ksnc_rx_iov[niov].iov_base = ksocknal_slop_buffer;
		conn->ksnc_rx_iov[niov].iov_len  = nob;
		niov++;
		skipped += nob;
		nob_to_skip -= nob;

	} while (nob_to_skip &&    /* mustn't overflow conn's rx iov */
		 niov < sizeof(conn->ksnc_rx_iov_space) / sizeof(struct iovec));

	conn->ksnc_rx_niov = niov;
	conn->ksnc_rx_kiov = NULL;
	conn->ksnc_rx_nkiov = 0;
	conn->ksnc_rx_nob_wanted = skipped;
	return 0;
}

static int
ksocknal_process_receive(struct ksock_conn *conn)
{
	struct lnet_hdr *lhdr;
	struct lnet_process_id *id;
	int rc;

	LASSERT(atomic_read(&conn->ksnc_conn_refcount) > 0);

	/* NB: sched lock NOT held */
	/* SOCKNAL_RX_LNET_HEADER is here for backward compatibility */
	LASSERT(conn->ksnc_rx_state == SOCKNAL_RX_KSM_HEADER ||
		conn->ksnc_rx_state == SOCKNAL_RX_LNET_PAYLOAD ||
		conn->ksnc_rx_state == SOCKNAL_RX_LNET_HEADER ||
		conn->ksnc_rx_state == SOCKNAL_RX_SLOP);
 again:
	if (conn->ksnc_rx_nob_wanted) {
		rc = ksocknal_receive(conn);

		if (rc <= 0) {
			LASSERT(rc != -EAGAIN);

			if (!rc)
				CDEBUG(D_NET, "[%p] EOF from %s ip %pI4h:%d\n",
				       conn,
				       libcfs_id2str(conn->ksnc_peer->ksnp_id),
				       &conn->ksnc_ipaddr,
				       conn->ksnc_port);
			else if (!conn->ksnc_closing)
				CERROR("[%p] Error %d on read from %s ip %pI4h:%d\n",
				       conn, rc,
				       libcfs_id2str(conn->ksnc_peer->ksnp_id),
				       &conn->ksnc_ipaddr,
				       conn->ksnc_port);

			/* it's not an error if conn is being closed */
			ksocknal_close_conn_and_siblings(conn,
							 (conn->ksnc_closing) ? 0 : rc);
			return (!rc ? -ESHUTDOWN : rc);
		}

		if (conn->ksnc_rx_nob_wanted) {
			/* short read */
			return -EAGAIN;
		}
	}
	switch (conn->ksnc_rx_state) {
	case SOCKNAL_RX_KSM_HEADER:
		if (conn->ksnc_flip) {
			__swab32s(&conn->ksnc_msg.ksm_type);
			__swab32s(&conn->ksnc_msg.ksm_csum);
			__swab64s(&conn->ksnc_msg.ksm_zc_cookies[0]);
			__swab64s(&conn->ksnc_msg.ksm_zc_cookies[1]);
		}

		if (conn->ksnc_msg.ksm_type != KSOCK_MSG_NOOP &&
		    conn->ksnc_msg.ksm_type != KSOCK_MSG_LNET) {
			CERROR("%s: Unknown message type: %x\n",
			       libcfs_id2str(conn->ksnc_peer->ksnp_id),
			       conn->ksnc_msg.ksm_type);
			ksocknal_new_packet(conn, 0);
			ksocknal_close_conn_and_siblings(conn, -EPROTO);
			return -EPROTO;
		}

		if (conn->ksnc_msg.ksm_type == KSOCK_MSG_NOOP &&
		    conn->ksnc_msg.ksm_csum &&     /* has checksum */
		    conn->ksnc_msg.ksm_csum != conn->ksnc_rx_csum) {
			/* NOOP Checksum error */
			CERROR("%s: Checksum error, wire:0x%08X data:0x%08X\n",
			       libcfs_id2str(conn->ksnc_peer->ksnp_id),
			       conn->ksnc_msg.ksm_csum, conn->ksnc_rx_csum);
			ksocknal_new_packet(conn, 0);
			ksocknal_close_conn_and_siblings(conn, -EPROTO);
			return -EIO;
		}

		if (conn->ksnc_msg.ksm_zc_cookies[1]) {
			__u64 cookie = 0;

			LASSERT(conn->ksnc_proto != &ksocknal_protocol_v1x);

			if (conn->ksnc_msg.ksm_type == KSOCK_MSG_NOOP)
				cookie = conn->ksnc_msg.ksm_zc_cookies[0];

			rc = conn->ksnc_proto->pro_handle_zcack(conn, cookie,
					       conn->ksnc_msg.ksm_zc_cookies[1]);

			if (rc) {
				CERROR("%s: Unknown ZC-ACK cookie: %llu, %llu\n",
				       libcfs_id2str(conn->ksnc_peer->ksnp_id),
				       cookie, conn->ksnc_msg.ksm_zc_cookies[1]);
				ksocknal_new_packet(conn, 0);
				ksocknal_close_conn_and_siblings(conn, -EPROTO);
				return rc;
			}
		}

		if (conn->ksnc_msg.ksm_type == KSOCK_MSG_NOOP) {
			ksocknal_new_packet(conn, 0);
			return 0;       /* NOOP is done and just return */
		}

		conn->ksnc_rx_state = SOCKNAL_RX_LNET_HEADER;
		conn->ksnc_rx_nob_wanted = sizeof(struct ksock_lnet_msg);
		conn->ksnc_rx_nob_left = sizeof(struct ksock_lnet_msg);

		conn->ksnc_rx_iov = (struct kvec *)&conn->ksnc_rx_iov_space;
		conn->ksnc_rx_iov[0].iov_base = &conn->ksnc_msg.ksm_u.lnetmsg;
		conn->ksnc_rx_iov[0].iov_len = sizeof(struct ksock_lnet_msg);

		conn->ksnc_rx_niov = 1;
		conn->ksnc_rx_kiov = NULL;
		conn->ksnc_rx_nkiov = 0;

		goto again;     /* read lnet header now */

	case SOCKNAL_RX_LNET_HEADER:
		/* unpack message header */
		conn->ksnc_proto->pro_unpack(&conn->ksnc_msg);

		if (conn->ksnc_peer->ksnp_id.pid & LNET_PID_USERFLAG) {
			/* Userspace peer */
			lhdr = &conn->ksnc_msg.ksm_u.lnetmsg.ksnm_hdr;
			id = &conn->ksnc_peer->ksnp_id;

			/* Substitute process ID assigned at connection time */
			lhdr->src_pid = cpu_to_le32(id->pid);
			lhdr->src_nid = cpu_to_le64(id->nid);
		}

		conn->ksnc_rx_state = SOCKNAL_RX_PARSE;
		ksocknal_conn_addref(conn);     /* ++ref while parsing */

		rc = lnet_parse(conn->ksnc_peer->ksnp_ni,
				&conn->ksnc_msg.ksm_u.lnetmsg.ksnm_hdr,
				conn->ksnc_peer->ksnp_id.nid, conn, 0);
		if (rc < 0) {
			/* I just received garbage: give up on this conn */
			ksocknal_new_packet(conn, 0);
			ksocknal_close_conn_and_siblings(conn, rc);
			ksocknal_conn_decref(conn);
			return -EPROTO;
		}

		/* I'm racing with ksocknal_recv() */
		LASSERT(conn->ksnc_rx_state == SOCKNAL_RX_PARSE ||
			conn->ksnc_rx_state == SOCKNAL_RX_LNET_PAYLOAD);

		if (conn->ksnc_rx_state != SOCKNAL_RX_LNET_PAYLOAD)
			return 0;

		/* ksocknal_recv() got called */
		goto again;

	case SOCKNAL_RX_LNET_PAYLOAD:
		/* payload all received */
		rc = 0;

		if (!conn->ksnc_rx_nob_left &&   /* not truncating */
		    conn->ksnc_msg.ksm_csum &&  /* has checksum */
		    conn->ksnc_msg.ksm_csum != conn->ksnc_rx_csum) {
			CERROR("%s: Checksum error, wire:0x%08X data:0x%08X\n",
			       libcfs_id2str(conn->ksnc_peer->ksnp_id),
			       conn->ksnc_msg.ksm_csum, conn->ksnc_rx_csum);
			rc = -EIO;
		}

		if (!rc && conn->ksnc_msg.ksm_zc_cookies[0]) {
			LASSERT(conn->ksnc_proto != &ksocknal_protocol_v1x);

			lhdr = &conn->ksnc_msg.ksm_u.lnetmsg.ksnm_hdr;
			id = &conn->ksnc_peer->ksnp_id;

			rc = conn->ksnc_proto->pro_handle_zcreq(conn,
					conn->ksnc_msg.ksm_zc_cookies[0],
					*ksocknal_tunables.ksnd_nonblk_zcack ||
					le64_to_cpu(lhdr->src_nid) != id->nid);
		}

		lnet_finalize(conn->ksnc_peer->ksnp_ni, conn->ksnc_cookie, rc);

		if (rc) {
			ksocknal_new_packet(conn, 0);
			ksocknal_close_conn_and_siblings(conn, rc);
			return -EPROTO;
		}
		/* Fall through */

	case SOCKNAL_RX_SLOP:
		/* starting new packet? */
		if (ksocknal_new_packet(conn, conn->ksnc_rx_nob_left))
			return 0;       /* come back later */
		goto again;	     /* try to finish reading slop now */

	default:
		break;
	}

	/* Not Reached */
	LBUG();
	return -EINVAL;		       /* keep gcc happy */
}

int
ksocknal_recv(struct lnet_ni *ni, void *private, struct lnet_msg *msg,
	      int delayed, struct iov_iter *to, unsigned int rlen)
{
	struct ksock_conn *conn = private;
	struct ksock_sched *sched = conn->ksnc_scheduler;

	LASSERT(iov_iter_count(to) <= rlen);
	LASSERT(to->nr_segs <= LNET_MAX_IOV);

	conn->ksnc_cookie = msg;
	conn->ksnc_rx_nob_wanted = iov_iter_count(to);
	conn->ksnc_rx_nob_left = rlen;

	if (to->type & ITER_KVEC) {
		conn->ksnc_rx_nkiov = 0;
		conn->ksnc_rx_kiov = NULL;
		conn->ksnc_rx_iov = conn->ksnc_rx_iov_space.iov;
		conn->ksnc_rx_niov =
			lnet_extract_iov(LNET_MAX_IOV, conn->ksnc_rx_iov,
					 to->nr_segs, to->kvec,
					 to->iov_offset, iov_iter_count(to));
	} else {
		conn->ksnc_rx_niov = 0;
		conn->ksnc_rx_iov = NULL;
		conn->ksnc_rx_kiov = conn->ksnc_rx_iov_space.kiov;
		conn->ksnc_rx_nkiov =
			lnet_extract_kiov(LNET_MAX_IOV, conn->ksnc_rx_kiov,
					 to->nr_segs, to->bvec,
					 to->iov_offset, iov_iter_count(to));
	}

	LASSERT(conn->ksnc_rx_scheduled);

	spin_lock_bh(&sched->kss_lock);

	switch (conn->ksnc_rx_state) {
	case SOCKNAL_RX_PARSE_WAIT:
		list_add_tail(&conn->ksnc_rx_list, &sched->kss_rx_conns);
		wake_up(&sched->kss_waitq);
		LASSERT(conn->ksnc_rx_ready);
		break;

	case SOCKNAL_RX_PARSE:
		/* scheduler hasn't noticed I'm parsing yet */
		break;
	}

	conn->ksnc_rx_state = SOCKNAL_RX_LNET_PAYLOAD;

	spin_unlock_bh(&sched->kss_lock);
	ksocknal_conn_decref(conn);
	return 0;
}

static inline int
ksocknal_sched_cansleep(struct ksock_sched *sched)
{
	int rc;

	spin_lock_bh(&sched->kss_lock);

	rc = !ksocknal_data.ksnd_shuttingdown &&
	      list_empty(&sched->kss_rx_conns) &&
	      list_empty(&sched->kss_tx_conns);

	spin_unlock_bh(&sched->kss_lock);
	return rc;
}

int ksocknal_scheduler(void *arg)
{
	struct ksock_sched_info *info;
	struct ksock_sched *sched;
	struct ksock_conn *conn;
	struct ksock_tx *tx;
	int rc;
	int nloops = 0;
	long id = (long)arg;

	info = ksocknal_data.ksnd_sched_info[KSOCK_THREAD_CPT(id)];
	sched = &info->ksi_scheds[KSOCK_THREAD_SID(id)];

	cfs_block_allsigs();

	rc = cfs_cpt_bind(lnet_cpt_table(), info->ksi_cpt);
	if (rc) {
		CWARN("Can't set CPU partition affinity to %d: %d\n",
		      info->ksi_cpt, rc);
	}

	spin_lock_bh(&sched->kss_lock);

	while (!ksocknal_data.ksnd_shuttingdown) {
		int did_something = 0;

		/* Ensure I progress everything semi-fairly */

		if (!list_empty(&sched->kss_rx_conns)) {
			conn = list_entry(sched->kss_rx_conns.next,
					  struct ksock_conn, ksnc_rx_list);
			list_del(&conn->ksnc_rx_list);

			LASSERT(conn->ksnc_rx_scheduled);
			LASSERT(conn->ksnc_rx_ready);

			/*
			 * clear rx_ready in case receive isn't complete.
			 * Do it BEFORE we call process_recv, since
			 * data_ready can set it any time after we release
			 * kss_lock.
			 */
			conn->ksnc_rx_ready = 0;
			spin_unlock_bh(&sched->kss_lock);

			rc = ksocknal_process_receive(conn);

			spin_lock_bh(&sched->kss_lock);

			/* I'm the only one that can clear this flag */
			LASSERT(conn->ksnc_rx_scheduled);

			/* Did process_receive get everything it wanted? */
			if (!rc)
				conn->ksnc_rx_ready = 1;

			if (conn->ksnc_rx_state == SOCKNAL_RX_PARSE) {
				/*
				 * Conn blocked waiting for ksocknal_recv()
				 * I change its state (under lock) to signal
				 * it can be rescheduled
				 */
				conn->ksnc_rx_state = SOCKNAL_RX_PARSE_WAIT;
			} else if (conn->ksnc_rx_ready) {
				/* reschedule for rx */
				list_add_tail(&conn->ksnc_rx_list,
					      &sched->kss_rx_conns);
			} else {
				conn->ksnc_rx_scheduled = 0;
				/* drop my ref */
				ksocknal_conn_decref(conn);
			}

			did_something = 1;
		}

		if (!list_empty(&sched->kss_tx_conns)) {
			LIST_HEAD(zlist);

			if (!list_empty(&sched->kss_zombie_noop_txs)) {
				list_add(&zlist, &sched->kss_zombie_noop_txs);
				list_del_init(&sched->kss_zombie_noop_txs);
			}

			conn = list_entry(sched->kss_tx_conns.next,
					  struct ksock_conn, ksnc_tx_list);
			list_del(&conn->ksnc_tx_list);

			LASSERT(conn->ksnc_tx_scheduled);
			LASSERT(conn->ksnc_tx_ready);
			LASSERT(!list_empty(&conn->ksnc_tx_queue));

			tx = list_entry(conn->ksnc_tx_queue.next,
					struct ksock_tx, tx_list);

			if (conn->ksnc_tx_carrier == tx)
				ksocknal_next_tx_carrier(conn);

			/* dequeue now so empty list => more to send */
			list_del(&tx->tx_list);

			/*
			 * Clear tx_ready in case send isn't complete.  Do
			 * it BEFORE we call process_transmit, since
			 * write_space can set it any time after we release
			 * kss_lock.
			 */
			conn->ksnc_tx_ready = 0;
			spin_unlock_bh(&sched->kss_lock);

			if (!list_empty(&zlist)) {
				/*
				 * free zombie noop txs, it's fast because
				 * noop txs are just put in freelist
				 */
				ksocknal_txlist_done(NULL, &zlist, 0);
			}

			rc = ksocknal_process_transmit(conn, tx);

			if (rc == -ENOMEM || rc == -EAGAIN) {
				/*
				 * Incomplete send: replace tx on HEAD of
				 * tx_queue
				 */
				spin_lock_bh(&sched->kss_lock);
				list_add(&tx->tx_list, &conn->ksnc_tx_queue);
			} else {
				/* Complete send; tx -ref */
				ksocknal_tx_decref(tx);

				spin_lock_bh(&sched->kss_lock);
				/* assume space for more */
				conn->ksnc_tx_ready = 1;
			}

			if (rc == -ENOMEM) {
				/*
				 * Do nothing; after a short timeout, this
				 * conn will be reposted on kss_tx_conns.
				 */
			} else if (conn->ksnc_tx_ready &&
				   !list_empty(&conn->ksnc_tx_queue)) {
				/* reschedule for tx */
				list_add_tail(&conn->ksnc_tx_list,
					      &sched->kss_tx_conns);
			} else {
				conn->ksnc_tx_scheduled = 0;
				/* drop my ref */
				ksocknal_conn_decref(conn);
			}

			did_something = 1;
		}
		if (!did_something ||	   /* nothing to do */
		    ++nloops == SOCKNAL_RESCHED) { /* hogging CPU? */
			spin_unlock_bh(&sched->kss_lock);

			nloops = 0;

			if (!did_something) {   /* wait for something to do */
				rc = wait_event_interruptible_exclusive(
					sched->kss_waitq,
					!ksocknal_sched_cansleep(sched));
				LASSERT(!rc);
			} else {
				cond_resched();
			}

			spin_lock_bh(&sched->kss_lock);
		}
	}

	spin_unlock_bh(&sched->kss_lock);
	ksocknal_thread_fini();
	return 0;
}

/*
 * Add connection to kss_rx_conns of scheduler
 * and wakeup the scheduler.
 */
void ksocknal_read_callback(struct ksock_conn *conn)
{
	struct ksock_sched *sched;

	sched = conn->ksnc_scheduler;

	spin_lock_bh(&sched->kss_lock);

	conn->ksnc_rx_ready = 1;

	if (!conn->ksnc_rx_scheduled) {  /* not being progressed */
		list_add_tail(&conn->ksnc_rx_list, &sched->kss_rx_conns);
		conn->ksnc_rx_scheduled = 1;
		/* extra ref for scheduler */
		ksocknal_conn_addref(conn);

		wake_up(&sched->kss_waitq);
	}
	spin_unlock_bh(&sched->kss_lock);
}

/*
 * Add connection to kss_tx_conns of scheduler
 * and wakeup the scheduler.
 */
void ksocknal_write_callback(struct ksock_conn *conn)
{
	struct ksock_sched *sched;

	sched = conn->ksnc_scheduler;

	spin_lock_bh(&sched->kss_lock);

	conn->ksnc_tx_ready = 1;

	if (!conn->ksnc_tx_scheduled && /* not being progressed */
	    !list_empty(&conn->ksnc_tx_queue)) { /* packets to send */
		list_add_tail(&conn->ksnc_tx_list, &sched->kss_tx_conns);
		conn->ksnc_tx_scheduled = 1;
		/* extra ref for scheduler */
		ksocknal_conn_addref(conn);

		wake_up(&sched->kss_waitq);
	}

	spin_unlock_bh(&sched->kss_lock);
}

static struct ksock_proto *
ksocknal_parse_proto_version(struct ksock_hello_msg *hello)
{
	__u32 version = 0;

	if (hello->kshm_magic == LNET_PROTO_MAGIC)
		version = hello->kshm_version;
	else if (hello->kshm_magic == __swab32(LNET_PROTO_MAGIC))
		version = __swab32(hello->kshm_version);

	if (version) {
#if SOCKNAL_VERSION_DEBUG
		if (*ksocknal_tunables.ksnd_protocol == 1)
			return NULL;

		if (*ksocknal_tunables.ksnd_protocol == 2 &&
		    version == KSOCK_PROTO_V3)
			return NULL;
#endif
		if (version == KSOCK_PROTO_V2)
			return &ksocknal_protocol_v2x;

		if (version == KSOCK_PROTO_V3)
			return &ksocknal_protocol_v3x;

		return NULL;
	}

	if (hello->kshm_magic == le32_to_cpu(LNET_PROTO_TCP_MAGIC)) {
		struct lnet_magicversion *hmv = (struct lnet_magicversion *)hello;

		BUILD_BUG_ON(sizeof(struct lnet_magicversion) !=
			     offsetof(struct ksock_hello_msg, kshm_src_nid));

		if (hmv->version_major == cpu_to_le16(KSOCK_PROTO_V1_MAJOR) &&
		    hmv->version_minor == cpu_to_le16(KSOCK_PROTO_V1_MINOR))
			return &ksocknal_protocol_v1x;
	}

	return NULL;
}

int
ksocknal_send_hello(struct lnet_ni *ni, struct ksock_conn *conn,
		    lnet_nid_t peer_nid, struct ksock_hello_msg *hello)
{
	/* CAVEAT EMPTOR: this byte flips 'ipaddrs' */
	struct ksock_net *net = (struct ksock_net *)ni->ni_data;

	LASSERT(hello->kshm_nips <= LNET_MAX_INTERFACES);

	/* rely on caller to hold a ref on socket so it wouldn't disappear */
	LASSERT(conn->ksnc_proto);

	hello->kshm_src_nid = ni->ni_nid;
	hello->kshm_dst_nid = peer_nid;
	hello->kshm_src_pid = the_lnet.ln_pid;

	hello->kshm_src_incarnation = net->ksnn_incarnation;
	hello->kshm_ctype = conn->ksnc_type;

	return conn->ksnc_proto->pro_send_hello(conn, hello);
}

static int
ksocknal_invert_type(int type)
{
	switch (type) {
	case SOCKLND_CONN_ANY:
	case SOCKLND_CONN_CONTROL:
		return type;
	case SOCKLND_CONN_BULK_IN:
		return SOCKLND_CONN_BULK_OUT;
	case SOCKLND_CONN_BULK_OUT:
		return SOCKLND_CONN_BULK_IN;
	default:
		return SOCKLND_CONN_NONE;
	}
}

int
ksocknal_recv_hello(struct lnet_ni *ni, struct ksock_conn *conn,
		    struct ksock_hello_msg *hello,
		    struct lnet_process_id *peerid,
		    __u64 *incarnation)
{
	/* Return < 0	fatal error
	 *	0	  success
	 *	EALREADY   lost connection race
	 *	EPROTO     protocol version mismatch
	 */
	struct socket *sock = conn->ksnc_sock;
	int active = !!conn->ksnc_proto;
	int timeout;
	int proto_match;
	int rc;
	struct ksock_proto *proto;
	struct lnet_process_id recv_id;

	/* socket type set on active connections - not set on passive */
	LASSERT(!active == !(conn->ksnc_type != SOCKLND_CONN_NONE));

	timeout = active ? *ksocknal_tunables.ksnd_timeout :
			    lnet_acceptor_timeout();

	rc = lnet_sock_read(sock, &hello->kshm_magic,
			    sizeof(hello->kshm_magic), timeout);
	if (rc) {
		CERROR("Error %d reading HELLO from %pI4h\n",
		       rc, &conn->ksnc_ipaddr);
		LASSERT(rc < 0);
		return rc;
	}

	if (hello->kshm_magic != LNET_PROTO_MAGIC &&
	    hello->kshm_magic != __swab32(LNET_PROTO_MAGIC) &&
	    hello->kshm_magic != le32_to_cpu(LNET_PROTO_TCP_MAGIC)) {
		/* Unexpected magic! */
		CERROR("Bad magic(1) %#08x (%#08x expected) from %pI4h\n",
		       __cpu_to_le32(hello->kshm_magic),
		       LNET_PROTO_TCP_MAGIC,
		       &conn->ksnc_ipaddr);
		return -EPROTO;
	}

	rc = lnet_sock_read(sock, &hello->kshm_version,
			    sizeof(hello->kshm_version), timeout);
	if (rc) {
		CERROR("Error %d reading HELLO from %pI4h\n",
		       rc, &conn->ksnc_ipaddr);
		LASSERT(rc < 0);
		return rc;
	}

	proto = ksocknal_parse_proto_version(hello);
	if (!proto) {
		if (!active) {
			/* unknown protocol from peer, tell peer my protocol */
			conn->ksnc_proto = &ksocknal_protocol_v3x;
#if SOCKNAL_VERSION_DEBUG
			if (*ksocknal_tunables.ksnd_protocol == 2)
				conn->ksnc_proto = &ksocknal_protocol_v2x;
			else if (*ksocknal_tunables.ksnd_protocol == 1)
				conn->ksnc_proto = &ksocknal_protocol_v1x;
#endif
			hello->kshm_nips = 0;
			ksocknal_send_hello(ni, conn, ni->ni_nid, hello);
		}

		CERROR("Unknown protocol version (%d.x expected) from %pI4h\n",
		       conn->ksnc_proto->pro_version,
		       &conn->ksnc_ipaddr);

		return -EPROTO;
	}

	proto_match = (conn->ksnc_proto == proto);
	conn->ksnc_proto = proto;

	/* receive the rest of hello message anyway */
	rc = conn->ksnc_proto->pro_recv_hello(conn, hello, timeout);
	if (rc) {
		CERROR("Error %d reading or checking hello from from %pI4h\n",
		       rc, &conn->ksnc_ipaddr);
		LASSERT(rc < 0);
		return rc;
	}

	*incarnation = hello->kshm_src_incarnation;

	if (hello->kshm_src_nid == LNET_NID_ANY) {
		CERROR("Expecting a HELLO hdr with a NID, but got LNET_NID_ANY from %pI4h\n",
		       &conn->ksnc_ipaddr);
		return -EPROTO;
	}

	if (!active &&
	    conn->ksnc_port > LNET_ACCEPTOR_MAX_RESERVED_PORT) {
		/* Userspace NAL assigns peer process ID from socket */
		recv_id.pid = conn->ksnc_port | LNET_PID_USERFLAG;
		recv_id.nid = LNET_MKNID(LNET_NIDNET(ni->ni_nid),
					 conn->ksnc_ipaddr);
	} else {
		recv_id.nid = hello->kshm_src_nid;
		recv_id.pid = hello->kshm_src_pid;
	}

	if (!active) {
		*peerid = recv_id;

		/* peer determines type */
		conn->ksnc_type = ksocknal_invert_type(hello->kshm_ctype);
		if (conn->ksnc_type == SOCKLND_CONN_NONE) {
			CERROR("Unexpected type %d from %s ip %pI4h\n",
			       hello->kshm_ctype, libcfs_id2str(*peerid),
			       &conn->ksnc_ipaddr);
			return -EPROTO;
		}

		return 0;
	}

	if (peerid->pid != recv_id.pid ||
	    peerid->nid != recv_id.nid) {
		LCONSOLE_ERROR_MSG(0x130, "Connected successfully to %s on host %pI4h, but they claimed they were %s; please check your Lustre configuration.\n",
				   libcfs_id2str(*peerid),
				   &conn->ksnc_ipaddr,
				   libcfs_id2str(recv_id));
		return -EPROTO;
	}

	if (hello->kshm_ctype == SOCKLND_CONN_NONE) {
		/* Possible protocol mismatch or I lost the connection race */
		return proto_match ? EALREADY : EPROTO;
	}

	if (ksocknal_invert_type(hello->kshm_ctype) != conn->ksnc_type) {
		CERROR("Mismatched types: me %d, %s ip %pI4h %d\n",
		       conn->ksnc_type, libcfs_id2str(*peerid),
		       &conn->ksnc_ipaddr, hello->kshm_ctype);
		return -EPROTO;
	}

	return 0;
}

static int
ksocknal_connect(struct ksock_route *route)
{
	LIST_HEAD(zombies);
	struct ksock_peer *peer = route->ksnr_peer;
	int type;
	int wanted;
	struct socket *sock;
	unsigned long deadline;
	int retry_later = 0;
	int rc = 0;

	deadline = cfs_time_add(cfs_time_current(),
				cfs_time_seconds(*ksocknal_tunables.ksnd_timeout));

	write_lock_bh(&ksocknal_data.ksnd_global_lock);

	LASSERT(route->ksnr_scheduled);
	LASSERT(!route->ksnr_connecting);

	route->ksnr_connecting = 1;

	for (;;) {
		wanted = ksocknal_route_mask() & ~route->ksnr_connected;

		/*
		 * stop connecting if peer/route got closed under me, or
		 * route got connected while queued
		 */
		if (peer->ksnp_closing || route->ksnr_deleted ||
		    !wanted) {
			retry_later = 0;
			break;
		}

		/* reschedule if peer is connecting to me */
		if (peer->ksnp_accepting > 0) {
			CDEBUG(D_NET,
			       "peer %s(%d) already connecting to me, retry later.\n",
			       libcfs_nid2str(peer->ksnp_id.nid),
			       peer->ksnp_accepting);
			retry_later = 1;
		}

		if (retry_later) /* needs reschedule */
			break;

		if (wanted & BIT(SOCKLND_CONN_ANY)) {
			type = SOCKLND_CONN_ANY;
		} else if (wanted & BIT(SOCKLND_CONN_CONTROL)) {
			type = SOCKLND_CONN_CONTROL;
		} else if (wanted & BIT(SOCKLND_CONN_BULK_IN)) {
			type = SOCKLND_CONN_BULK_IN;
		} else {
			LASSERT(wanted & BIT(SOCKLND_CONN_BULK_OUT));
			type = SOCKLND_CONN_BULK_OUT;
		}

		write_unlock_bh(&ksocknal_data.ksnd_global_lock);

		if (cfs_time_aftereq(cfs_time_current(), deadline)) {
			rc = -ETIMEDOUT;
			lnet_connect_console_error(rc, peer->ksnp_id.nid,
						   route->ksnr_ipaddr,
						   route->ksnr_port);
			goto failed;
		}

		rc = lnet_connect(&sock, peer->ksnp_id.nid,
				  route->ksnr_myipaddr,
				  route->ksnr_ipaddr, route->ksnr_port);
		if (rc)
			goto failed;

		rc = ksocknal_create_conn(peer->ksnp_ni, route, sock, type);
		if (rc < 0) {
			lnet_connect_console_error(rc, peer->ksnp_id.nid,
						   route->ksnr_ipaddr,
						   route->ksnr_port);
			goto failed;
		}

		/*
		 * A +ve RC means I have to retry because I lost the connection
		 * race or I have to renegotiate protocol version
		 */
		retry_later = (rc);
		if (retry_later)
			CDEBUG(D_NET, "peer %s: conn race, retry later.\n",
			       libcfs_nid2str(peer->ksnp_id.nid));

		write_lock_bh(&ksocknal_data.ksnd_global_lock);
	}

	route->ksnr_scheduled = 0;
	route->ksnr_connecting = 0;

	if (retry_later) {
		/*
		 * re-queue for attention; this frees me up to handle
		 * the peer's incoming connection request
		 */
		if (rc == EALREADY ||
		    (!rc && peer->ksnp_accepting > 0)) {
			/*
			 * We want to introduce a delay before next
			 * attempt to connect if we lost conn race,
			 * but the race is resolved quickly usually,
			 * so min_reconnectms should be good heuristic
			 */
			route->ksnr_retry_interval =
				cfs_time_seconds(*ksocknal_tunables.ksnd_min_reconnectms) / 1000;
			route->ksnr_timeout = cfs_time_add(cfs_time_current(),
							   route->ksnr_retry_interval);
		}

		ksocknal_launch_connection_locked(route);
	}

	write_unlock_bh(&ksocknal_data.ksnd_global_lock);
	return retry_later;

 failed:
	write_lock_bh(&ksocknal_data.ksnd_global_lock);

	route->ksnr_scheduled = 0;
	route->ksnr_connecting = 0;

	/* This is a retry rather than a new connection */
	route->ksnr_retry_interval *= 2;
	route->ksnr_retry_interval =
		max(route->ksnr_retry_interval,
		    cfs_time_seconds(*ksocknal_tunables.ksnd_min_reconnectms) / 1000);
	route->ksnr_retry_interval =
		min(route->ksnr_retry_interval,
		    cfs_time_seconds(*ksocknal_tunables.ksnd_max_reconnectms) / 1000);

	LASSERT(route->ksnr_retry_interval);
	route->ksnr_timeout = cfs_time_add(cfs_time_current(),
					   route->ksnr_retry_interval);

	if (!list_empty(&peer->ksnp_tx_queue) &&
	    !peer->ksnp_accepting &&
	    !ksocknal_find_connecting_route_locked(peer)) {
		struct ksock_conn *conn;

		/*
		 * ksnp_tx_queue is queued on a conn on successful
		 * connection for V1.x and V2.x
		 */
		if (!list_empty(&peer->ksnp_conns)) {
			conn = list_entry(peer->ksnp_conns.next,
					  struct ksock_conn, ksnc_list);
			LASSERT(conn->ksnc_proto == &ksocknal_protocol_v3x);
		}

		/*
		 * take all the blocked packets while I've got the lock and
		 * complete below...
		 */
		list_splice_init(&peer->ksnp_tx_queue, &zombies);
	}

	write_unlock_bh(&ksocknal_data.ksnd_global_lock);

	ksocknal_peer_failed(peer);
	ksocknal_txlist_done(peer->ksnp_ni, &zombies, 1);
	return 0;
}

/*
 * check whether we need to create more connds.
 * It will try to create new thread if it's necessary, @timeout can
 * be updated if failed to create, so caller wouldn't keep try while
 * running out of resource.
 */
static int
ksocknal_connd_check_start(time64_t sec, long *timeout)
{
	char name[16];
	int rc;
	int total = ksocknal_data.ksnd_connd_starting +
		    ksocknal_data.ksnd_connd_running;

	if (unlikely(ksocknal_data.ksnd_init < SOCKNAL_INIT_ALL)) {
		/* still in initializing */
		return 0;
	}

	if (total >= *ksocknal_tunables.ksnd_nconnds_max ||
	    total > ksocknal_data.ksnd_connd_connecting + SOCKNAL_CONND_RESV) {
		/*
		 * can't create more connd, or still have enough
		 * threads to handle more connecting
		 */
		return 0;
	}

	if (list_empty(&ksocknal_data.ksnd_connd_routes)) {
		/* no pending connecting request */
		return 0;
	}

	if (sec - ksocknal_data.ksnd_connd_failed_stamp <= 1) {
		/* may run out of resource, retry later */
		*timeout = cfs_time_seconds(1);
		return 0;
	}

	if (ksocknal_data.ksnd_connd_starting > 0) {
		/* serialize starting to avoid flood */
		return 0;
	}

	ksocknal_data.ksnd_connd_starting_stamp = sec;
	ksocknal_data.ksnd_connd_starting++;
	spin_unlock_bh(&ksocknal_data.ksnd_connd_lock);

	/* NB: total is the next id */
	snprintf(name, sizeof(name), "socknal_cd%02d", total);
	rc = ksocknal_thread_start(ksocknal_connd, NULL, name);

	spin_lock_bh(&ksocknal_data.ksnd_connd_lock);
	if (!rc)
		return 1;

	/* we tried ... */
	LASSERT(ksocknal_data.ksnd_connd_starting > 0);
	ksocknal_data.ksnd_connd_starting--;
	ksocknal_data.ksnd_connd_failed_stamp = ktime_get_real_seconds();

	return 1;
}

/*
 * check whether current thread can exit, it will return 1 if there are too
 * many threads and no creating in past 120 seconds.
 * Also, this function may update @timeout to make caller come back
 * again to recheck these conditions.
 */
static int
ksocknal_connd_check_stop(time64_t sec, long *timeout)
{
	int val;

	if (unlikely(ksocknal_data.ksnd_init < SOCKNAL_INIT_ALL)) {
		/* still in initializing */
		return 0;
	}

	if (ksocknal_data.ksnd_connd_starting > 0) {
		/* in progress of starting new thread */
		return 0;
	}

	if (ksocknal_data.ksnd_connd_running <=
	    *ksocknal_tunables.ksnd_nconnds) { /* can't shrink */
		return 0;
	}

	/* created thread in past 120 seconds? */
	val = (int)(ksocknal_data.ksnd_connd_starting_stamp +
		    SOCKNAL_CONND_TIMEOUT - sec);

	*timeout = (val > 0) ? cfs_time_seconds(val) :
			       cfs_time_seconds(SOCKNAL_CONND_TIMEOUT);
	if (val > 0)
		return 0;

	/* no creating in past 120 seconds */

	return ksocknal_data.ksnd_connd_running >
	       ksocknal_data.ksnd_connd_connecting + SOCKNAL_CONND_RESV;
}

/*
 * Go through connd_routes queue looking for a route that we can process
 * right now, @timeout_p can be updated if we need to come back later
 */
static struct ksock_route *
ksocknal_connd_get_route_locked(signed long *timeout_p)
{
	struct ksock_route *route;
	unsigned long now;

	now = cfs_time_current();

	/* connd_routes can contain both pending and ordinary routes */
	list_for_each_entry(route, &ksocknal_data.ksnd_connd_routes,
			    ksnr_connd_list) {
		if (!route->ksnr_retry_interval ||
		    cfs_time_aftereq(now, route->ksnr_timeout))
			return route;

		if (*timeout_p == MAX_SCHEDULE_TIMEOUT ||
		    (int)*timeout_p > (int)(route->ksnr_timeout - now))
			*timeout_p = (int)(route->ksnr_timeout - now);
	}

	return NULL;
}

int
ksocknal_connd(void *arg)
{
	spinlock_t *connd_lock = &ksocknal_data.ksnd_connd_lock;
	struct ksock_connreq *cr;
	wait_queue_entry_t wait;
	int nloops = 0;
	int cons_retry = 0;

	cfs_block_allsigs();

	init_waitqueue_entry(&wait, current);

	spin_lock_bh(connd_lock);

	LASSERT(ksocknal_data.ksnd_connd_starting > 0);
	ksocknal_data.ksnd_connd_starting--;
	ksocknal_data.ksnd_connd_running++;

	while (!ksocknal_data.ksnd_shuttingdown) {
		struct ksock_route *route = NULL;
		time64_t sec = ktime_get_real_seconds();
		long timeout = MAX_SCHEDULE_TIMEOUT;
		int dropped_lock = 0;

		if (ksocknal_connd_check_stop(sec, &timeout)) {
			/* wakeup another one to check stop */
			wake_up(&ksocknal_data.ksnd_connd_waitq);
			break;
		}

		if (ksocknal_connd_check_start(sec, &timeout)) {
			/* created new thread */
			dropped_lock = 1;
		}

		if (!list_empty(&ksocknal_data.ksnd_connd_connreqs)) {
			/* Connection accepted by the listener */
			cr = list_entry(ksocknal_data.ksnd_connd_connreqs.next,
					struct ksock_connreq, ksncr_list);

			list_del(&cr->ksncr_list);
			spin_unlock_bh(connd_lock);
			dropped_lock = 1;

			ksocknal_create_conn(cr->ksncr_ni, NULL,
					     cr->ksncr_sock, SOCKLND_CONN_NONE);
			lnet_ni_decref(cr->ksncr_ni);
			LIBCFS_FREE(cr, sizeof(*cr));

			spin_lock_bh(connd_lock);
		}

		/*
		 * Only handle an outgoing connection request if there
		 * is a thread left to handle incoming connections and
		 * create new connd
		 */
		if (ksocknal_data.ksnd_connd_connecting + SOCKNAL_CONND_RESV <
		    ksocknal_data.ksnd_connd_running) {
			route = ksocknal_connd_get_route_locked(&timeout);
		}
		if (route) {
			list_del(&route->ksnr_connd_list);
			ksocknal_data.ksnd_connd_connecting++;
			spin_unlock_bh(connd_lock);
			dropped_lock = 1;

			if (ksocknal_connect(route)) {
				/* consecutive retry */
				if (cons_retry++ > SOCKNAL_INSANITY_RECONN) {
					CWARN("massive consecutive re-connecting to %pI4h\n",
					      &route->ksnr_ipaddr);
					cons_retry = 0;
				}
			} else {
				cons_retry = 0;
			}

			ksocknal_route_decref(route);

			spin_lock_bh(connd_lock);
			ksocknal_data.ksnd_connd_connecting--;
		}

		if (dropped_lock) {
			if (++nloops < SOCKNAL_RESCHED)
				continue;
			spin_unlock_bh(connd_lock);
			nloops = 0;
			cond_resched();
			spin_lock_bh(connd_lock);
			continue;
		}

		/* Nothing to do for 'timeout'  */
		set_current_state(TASK_INTERRUPTIBLE);
		add_wait_queue_exclusive(&ksocknal_data.ksnd_connd_waitq,
					 &wait);
		spin_unlock_bh(connd_lock);

		nloops = 0;
		schedule_timeout(timeout);

		remove_wait_queue(&ksocknal_data.ksnd_connd_waitq, &wait);
		spin_lock_bh(connd_lock);
	}
	ksocknal_data.ksnd_connd_running--;
	spin_unlock_bh(connd_lock);

	ksocknal_thread_fini();
	return 0;
}

static struct ksock_conn *
ksocknal_find_timed_out_conn(struct ksock_peer *peer)
{
	/* We're called with a shared lock on ksnd_global_lock */
	struct ksock_conn *conn;
	struct list_head *ctmp;

	list_for_each(ctmp, &peer->ksnp_conns) {
		int error;

		conn = list_entry(ctmp, struct ksock_conn, ksnc_list);

		/* Don't need the {get,put}connsock dance to deref ksnc_sock */
		LASSERT(!conn->ksnc_closing);

		/*
		 * SOCK_ERROR will reset error code of socket in
		 * some platform (like Darwin8.x)
		 */
		error = conn->ksnc_sock->sk->sk_err;
		if (error) {
			ksocknal_conn_addref(conn);

			switch (error) {
			case ECONNRESET:
				CNETERR("A connection with %s (%pI4h:%d) was reset; it may have rebooted.\n",
					libcfs_id2str(peer->ksnp_id),
					&conn->ksnc_ipaddr,
					conn->ksnc_port);
				break;
			case ETIMEDOUT:
				CNETERR("A connection with %s (%pI4h:%d) timed out; the network or node may be down.\n",
					libcfs_id2str(peer->ksnp_id),
					&conn->ksnc_ipaddr,
					conn->ksnc_port);
				break;
			default:
				CNETERR("An unexpected network error %d occurred with %s (%pI4h:%d\n",
					error,
					libcfs_id2str(peer->ksnp_id),
					&conn->ksnc_ipaddr,
					conn->ksnc_port);
				break;
			}

			return conn;
		}

		if (conn->ksnc_rx_started &&
		    cfs_time_aftereq(cfs_time_current(),
				     conn->ksnc_rx_deadline)) {
			/* Timed out incomplete incoming message */
			ksocknal_conn_addref(conn);
			CNETERR("Timeout receiving from %s (%pI4h:%d), state %d wanted %d left %d\n",
				libcfs_id2str(peer->ksnp_id),
				&conn->ksnc_ipaddr,
				conn->ksnc_port,
				conn->ksnc_rx_state,
				conn->ksnc_rx_nob_wanted,
				conn->ksnc_rx_nob_left);
			return conn;
		}

		if ((!list_empty(&conn->ksnc_tx_queue) ||
		     conn->ksnc_sock->sk->sk_wmem_queued) &&
		    cfs_time_aftereq(cfs_time_current(),
				     conn->ksnc_tx_deadline)) {
			/*
			 * Timed out messages queued for sending or
			 * buffered in the socket's send buffer
			 */
			ksocknal_conn_addref(conn);
			CNETERR("Timeout sending data to %s (%pI4h:%d) the network or that node may be down.\n",
				libcfs_id2str(peer->ksnp_id),
				&conn->ksnc_ipaddr,
				conn->ksnc_port);
			return conn;
		}
	}

	return NULL;
}

static inline void
ksocknal_flush_stale_txs(struct ksock_peer *peer)
{
	struct ksock_tx *tx;
	struct ksock_tx *tmp;
	LIST_HEAD(stale_txs);

	write_lock_bh(&ksocknal_data.ksnd_global_lock);

	list_for_each_entry_safe(tx, tmp, &peer->ksnp_tx_queue, tx_list) {
		if (!cfs_time_aftereq(cfs_time_current(),
				      tx->tx_deadline))
			break;

		list_del(&tx->tx_list);
		list_add_tail(&tx->tx_list, &stale_txs);
	}

	write_unlock_bh(&ksocknal_data.ksnd_global_lock);

	ksocknal_txlist_done(peer->ksnp_ni, &stale_txs, 1);
}

static int
ksocknal_send_keepalive_locked(struct ksock_peer *peer)
	__must_hold(&ksocknal_data.ksnd_global_lock)
{
	struct ksock_sched *sched;
	struct ksock_conn *conn;
	struct ksock_tx *tx;

	/* last_alive will be updated by create_conn */
	if (list_empty(&peer->ksnp_conns))
		return 0;

	if (peer->ksnp_proto != &ksocknal_protocol_v3x)
		return 0;

	if (*ksocknal_tunables.ksnd_keepalive <= 0 ||
	    time_before(cfs_time_current(),
			cfs_time_add(peer->ksnp_last_alive,
				     cfs_time_seconds(*ksocknal_tunables.ksnd_keepalive))))
		return 0;

	if (time_before(cfs_time_current(), peer->ksnp_send_keepalive))
		return 0;

	/*
	 * retry 10 secs later, so we wouldn't put pressure
	 * on this peer if we failed to send keepalive this time
	 */
	peer->ksnp_send_keepalive = cfs_time_shift(10);

	conn = ksocknal_find_conn_locked(peer, NULL, 1);
	if (conn) {
		sched = conn->ksnc_scheduler;

		spin_lock_bh(&sched->kss_lock);
		if (!list_empty(&conn->ksnc_tx_queue)) {
			spin_unlock_bh(&sched->kss_lock);
			/* there is an queued ACK, don't need keepalive */
			return 0;
		}

		spin_unlock_bh(&sched->kss_lock);
	}

	read_unlock(&ksocknal_data.ksnd_global_lock);

	/* cookie = 1 is reserved for keepalive PING */
	tx = ksocknal_alloc_tx_noop(1, 1);
	if (!tx) {
		read_lock(&ksocknal_data.ksnd_global_lock);
		return -ENOMEM;
	}

	if (!ksocknal_launch_packet(peer->ksnp_ni, tx, peer->ksnp_id)) {
		read_lock(&ksocknal_data.ksnd_global_lock);
		return 1;
	}

	ksocknal_free_tx(tx);
	read_lock(&ksocknal_data.ksnd_global_lock);

	return -EIO;
}

static void
ksocknal_check_peer_timeouts(int idx)
{
	struct list_head *peers = &ksocknal_data.ksnd_peers[idx];
	struct ksock_peer *peer;
	struct ksock_conn *conn;
	struct ksock_tx *tx;

 again:
	/*
	 * NB. We expect to have a look at all the peers and not find any
	 * connections to time out, so we just use a shared lock while we
	 * take a look...
	 */
	read_lock(&ksocknal_data.ksnd_global_lock);

	list_for_each_entry(peer, peers, ksnp_list) {
		unsigned long deadline = 0;
		struct ksock_tx *tx_stale;
		int resid = 0;
		int n = 0;

		if (ksocknal_send_keepalive_locked(peer)) {
			read_unlock(&ksocknal_data.ksnd_global_lock);
			goto again;
		}

		conn = ksocknal_find_timed_out_conn(peer);

		if (conn) {
			read_unlock(&ksocknal_data.ksnd_global_lock);

			ksocknal_close_conn_and_siblings(conn, -ETIMEDOUT);

			/*
			 * NB we won't find this one again, but we can't
			 * just proceed with the next peer, since we dropped
			 * ksnd_global_lock and it might be dead already!
			 */
			ksocknal_conn_decref(conn);
			goto again;
		}

		/*
		 * we can't process stale txs right here because we're
		 * holding only shared lock
		 */
		if (!list_empty(&peer->ksnp_tx_queue)) {
			tx = list_entry(peer->ksnp_tx_queue.next,
					struct ksock_tx, tx_list);

			if (cfs_time_aftereq(cfs_time_current(),
					     tx->tx_deadline)) {
				ksocknal_peer_addref(peer);
				read_unlock(&ksocknal_data.ksnd_global_lock);

				ksocknal_flush_stale_txs(peer);

				ksocknal_peer_decref(peer);
				goto again;
			}
		}

		if (list_empty(&peer->ksnp_zc_req_list))
			continue;

		tx_stale = NULL;
		spin_lock(&peer->ksnp_lock);
		list_for_each_entry(tx, &peer->ksnp_zc_req_list, tx_zc_list) {
			if (!cfs_time_aftereq(cfs_time_current(),
					      tx->tx_deadline))
				break;
			/* ignore the TX if connection is being closed */
			if (tx->tx_conn->ksnc_closing)
				continue;
			if (!tx_stale)
				tx_stale = tx;
			n++;
		}

		if (!tx_stale) {
			spin_unlock(&peer->ksnp_lock);
			continue;
		}

		deadline = tx_stale->tx_deadline;
		resid = tx_stale->tx_resid;
		conn = tx_stale->tx_conn;
		ksocknal_conn_addref(conn);

		spin_unlock(&peer->ksnp_lock);
		read_unlock(&ksocknal_data.ksnd_global_lock);

		CERROR("Total %d stale ZC_REQs for peer %s detected; the oldest(%p) timed out %ld secs ago, resid: %d, wmem: %d\n",
		       n, libcfs_nid2str(peer->ksnp_id.nid), tx_stale,
		       cfs_duration_sec(cfs_time_current() - deadline),
		       resid, conn->ksnc_sock->sk->sk_wmem_queued);

		ksocknal_close_conn_and_siblings(conn, -ETIMEDOUT);
		ksocknal_conn_decref(conn);
		goto again;
	}

	read_unlock(&ksocknal_data.ksnd_global_lock);
}

int
ksocknal_reaper(void *arg)
{
	wait_queue_entry_t wait;
	struct ksock_conn *conn;
	struct ksock_sched *sched;
	struct list_head enomem_conns;
	int nenomem_conns;
	long timeout;
	int i;
	int peer_index = 0;
	unsigned long deadline = cfs_time_current();

	cfs_block_allsigs();

	INIT_LIST_HEAD(&enomem_conns);
	init_waitqueue_entry(&wait, current);

	spin_lock_bh(&ksocknal_data.ksnd_reaper_lock);

	while (!ksocknal_data.ksnd_shuttingdown) {
		if (!list_empty(&ksocknal_data.ksnd_deathrow_conns)) {
			conn = list_entry(ksocknal_data.ksnd_deathrow_conns.next,
					  struct ksock_conn, ksnc_list);
			list_del(&conn->ksnc_list);

			spin_unlock_bh(&ksocknal_data.ksnd_reaper_lock);

			ksocknal_terminate_conn(conn);
			ksocknal_conn_decref(conn);

			spin_lock_bh(&ksocknal_data.ksnd_reaper_lock);
			continue;
		}

		if (!list_empty(&ksocknal_data.ksnd_zombie_conns)) {
			conn = list_entry(ksocknal_data.ksnd_zombie_conns.next,
					  struct ksock_conn, ksnc_list);
			list_del(&conn->ksnc_list);

			spin_unlock_bh(&ksocknal_data.ksnd_reaper_lock);

			ksocknal_destroy_conn(conn);

			spin_lock_bh(&ksocknal_data.ksnd_reaper_lock);
			continue;
		}

		if (!list_empty(&ksocknal_data.ksnd_enomem_conns)) {
			list_add(&enomem_conns,
				 &ksocknal_data.ksnd_enomem_conns);
			list_del_init(&ksocknal_data.ksnd_enomem_conns);
		}

		spin_unlock_bh(&ksocknal_data.ksnd_reaper_lock);

		/* reschedule all the connections that stalled with ENOMEM... */
		nenomem_conns = 0;
		while (!list_empty(&enomem_conns)) {
			conn = list_entry(enomem_conns.next, struct ksock_conn,
					  ksnc_tx_list);
			list_del(&conn->ksnc_tx_list);

			sched = conn->ksnc_scheduler;

			spin_lock_bh(&sched->kss_lock);

			LASSERT(conn->ksnc_tx_scheduled);
			conn->ksnc_tx_ready = 1;
			list_add_tail(&conn->ksnc_tx_list,
				      &sched->kss_tx_conns);
			wake_up(&sched->kss_waitq);

			spin_unlock_bh(&sched->kss_lock);
			nenomem_conns++;
		}

		/* careful with the jiffy wrap... */
		while ((timeout = cfs_time_sub(deadline,
					       cfs_time_current())) <= 0) {
			const int n = 4;
			const int p = 1;
			int chunk = ksocknal_data.ksnd_peer_hash_size;

			/*
			 * Time to check for timeouts on a few more peers: I do
			 * checks every 'p' seconds on a proportion of the peer
			 * table and I need to check every connection 'n' times
			 * within a timeout interval, to ensure I detect a
			 * timeout on any connection within (n+1)/n times the
			 * timeout interval.
			 */
			if (*ksocknal_tunables.ksnd_timeout > n * p)
				chunk = (chunk * n * p) /
					*ksocknal_tunables.ksnd_timeout;
			if (!chunk)
				chunk = 1;

			for (i = 0; i < chunk; i++) {
				ksocknal_check_peer_timeouts(peer_index);
				peer_index = (peer_index + 1) %
					     ksocknal_data.ksnd_peer_hash_size;
			}

			deadline = cfs_time_add(deadline, cfs_time_seconds(p));
		}

		if (nenomem_conns) {
			/*
			 * Reduce my timeout if I rescheduled ENOMEM conns.
			 * This also prevents me getting woken immediately
			 * if any go back on my enomem list.
			 */
			timeout = SOCKNAL_ENOMEM_RETRY;
		}
		ksocknal_data.ksnd_reaper_waketime =
			cfs_time_add(cfs_time_current(), timeout);

		set_current_state(TASK_INTERRUPTIBLE);
		add_wait_queue(&ksocknal_data.ksnd_reaper_waitq, &wait);

		if (!ksocknal_data.ksnd_shuttingdown &&
		    list_empty(&ksocknal_data.ksnd_deathrow_conns) &&
		    list_empty(&ksocknal_data.ksnd_zombie_conns))
			schedule_timeout(timeout);

		set_current_state(TASK_RUNNING);
		remove_wait_queue(&ksocknal_data.ksnd_reaper_waitq, &wait);

		spin_lock_bh(&ksocknal_data.ksnd_reaper_lock);
	}

	spin_unlock_bh(&ksocknal_data.ksnd_reaper_lock);

	ksocknal_thread_fini();
	return 0;
}