xref: /illumos-gate/usr/src/uts/common/io/mac/mac_util.c (revision c61a1653)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2019 Joyent, Inc.
 */

/*
 * MAC Services Module - misc utilities
 */

#include <sys/types.h>
#include <sys/mac.h>
#include <sys/mac_impl.h>
#include <sys/mac_client_priv.h>
#include <sys/mac_client_impl.h>
#include <sys/mac_soft_ring.h>
#include <sys/strsubr.h>
#include <sys/strsun.h>
#include <sys/vlan.h>
#include <sys/pattr.h>
#include <sys/pci_tools.h>
#include <inet/ip.h>
#include <inet/ip_impl.h>
#include <inet/ip6.h>
#include <sys/vtrace.h>
#include <sys/dlpi.h>
#include <sys/sunndi.h>
#include <inet/ipsec_impl.h>
#include <inet/sadb.h>
#include <inet/ipsecesp.h>
#include <inet/ipsecah.h>
#include <inet/tcp.h>
#include <inet/udp_impl.h>
#include <inet/sctp_ip.h>

/*
 * The next two functions are used for dropping packets or chains of
 * packets, respectively. We could use one function for both but
 * separating the use cases allows us to specify intent and prevent
 * dropping more data than intended.
 *
 * The purpose of these functions is to aid the debugging effort,
 * especially in production. Rather than use freemsg()/freemsgchain(),
 * it's preferable to use these functions when dropping a packet in
 * the MAC layer. These functions should only be used during
 * unexpected conditions. That is, any time a packet is dropped
 * outside of the regular, successful datapath. Consolidating all
 * drops on these functions allows the user to trace one location and
 * determine why the packet was dropped based on the msg. It also
 * allows the user to inspect the packet before it is freed. Finally,
 * it allows the user to avoid tracing freemsg()/freemsgchain() thus
 * keeping the hot path running as efficiently as possible.
 *
 * NOTE: At this time not all MAC drops are aggregated on these
 * functions; but that is the plan. This comment should be erased once
 * completed.
 */

/*PRINTFLIKE2*/
void
mac_drop_pkt(mblk_t *mp, const char *fmt, ...)
{
	va_list adx;
	char msg[128];
	char *msgp = msg;

	ASSERT3P(mp->b_next, ==, NULL);

	va_start(adx, fmt);
	(void) vsnprintf(msgp, sizeof (msg), fmt, adx);
	va_end(adx);

	DTRACE_PROBE2(mac__drop, mblk_t *, mp, char *, msgp);
	freemsg(mp);
}

/*PRINTFLIKE2*/
void
mac_drop_chain(mblk_t *chain, const char *fmt, ...)
{
	va_list adx;
	char msg[128];
	char *msgp = msg;

	va_start(adx, fmt);
	(void) vsnprintf(msgp, sizeof (msg), fmt, adx);
	va_end(adx);

	/*
	 * We could use freemsgchain() for the actual freeing but
	 * since we are already walking the chain to fire the dtrace
	 * probe we might as well free the msg here too.
	 */
	for (mblk_t *mp = chain, *next; mp != NULL; ) {
		next = mp->b_next;
		DTRACE_PROBE2(mac__drop, mblk_t *, mp, char *, msgp);
		freemsg(mp);
		mp = next;
	}
}

/*
 * Copy an mblk, preserving its hardware checksum flags.
 */
static mblk_t *
mac_copymsg_cksum(mblk_t *mp)
{
	mblk_t *mp1;

	mp1 = copymsg(mp);
	if (mp1 == NULL)
		return (NULL);

	mac_hcksum_clone(mp, mp1);

	return (mp1);
}

/*
 * Copy an mblk chain, presenting the hardware checksum flags of the
 * individual mblks.
 */
mblk_t *
mac_copymsgchain_cksum(mblk_t *mp)
{
	mblk_t *nmp = NULL;
	mblk_t **nmpp = &nmp;

	for (; mp != NULL; mp = mp->b_next) {
		if ((*nmpp = mac_copymsg_cksum(mp)) == NULL) {
			freemsgchain(nmp);
			return (NULL);
		}

		nmpp = &((*nmpp)->b_next);
	}

	return (nmp);
}

/*
 * Calculate the ULP checksum for IPv4. Return true if the calculation
 * was successful, or false if an error occurred. If the later, place
 * an error message into '*err'.
 */
static boolean_t
mac_sw_cksum_ipv4(mblk_t *mp, uint32_t ip_hdr_offset, ipha_t *ipha,
    const char **err)
{
	const uint8_t proto = ipha->ipha_protocol;
	size_t len;
	const uint32_t ip_hdr_sz = IPH_HDR_LENGTH(ipha);
	/* ULP offset from start of L2. */
	const uint32_t ulp_offset = ip_hdr_offset + ip_hdr_sz;
	ipaddr_t src, dst;
	uint32_t cksum;
	uint16_t *up;

	/*
	 * We need a pointer to the ULP checksum. We're assuming the
	 * ULP checksum pointer resides in the first mblk. Our native
	 * TCP stack should always put the headers in the first mblk,
	 * but currently we have no way to guarantee that other
	 * clients don't spread headers (or even header fields) across
	 * mblks.
	 */
	switch (proto) {
	case IPPROTO_TCP:
		ASSERT3U(MBLKL(mp), >=, (ulp_offset + sizeof (tcph_t)));
		if (MBLKL(mp) < (ulp_offset + sizeof (tcph_t))) {
			*err = "mblk doesn't contain TCP header";
			goto bail;
		}

		up = IPH_TCPH_CHECKSUMP(ipha, ip_hdr_sz);
		cksum = IP_TCP_CSUM_COMP;
		break;

	case IPPROTO_UDP:
		ASSERT3U(MBLKL(mp), >=, (ulp_offset + sizeof (udpha_t)));
		if (MBLKL(mp) < (ulp_offset + sizeof (udpha_t))) {
			*err = "mblk doesn't contain UDP header";
			goto bail;
		}

		up = IPH_UDPH_CHECKSUMP(ipha, ip_hdr_sz);
		cksum = IP_UDP_CSUM_COMP;
		break;

	case IPPROTO_SCTP: {
		sctp_hdr_t *sctph;

		ASSERT3U(MBLKL(mp), >=, (ulp_offset + sizeof (sctp_hdr_t)));
		if (MBLKL(mp) < (ulp_offset + sizeof (sctp_hdr_t))) {
			*err = "mblk doesn't contain SCTP header";
			goto bail;
		}

		sctph = (sctp_hdr_t *)(mp->b_rptr + ulp_offset);
		sctph->sh_chksum = 0;
		sctph->sh_chksum = sctp_cksum(mp, ulp_offset);
		return (B_TRUE);
	}

	default:
		*err = "unexpected protocol";
		goto bail;

	}

	/* Pseudo-header checksum. */
	src = ipha->ipha_src;
	dst = ipha->ipha_dst;
	len = ntohs(ipha->ipha_length) - ip_hdr_sz;

	cksum += (dst >> 16) + (dst & 0xFFFF) + (src >> 16) + (src & 0xFFFF);
	cksum += htons(len);

	/*
	 * We have already accounted for the pseudo checksum above.
	 * Make sure the ULP checksum field is zero before computing
	 * the rest.
	 */
	*up = 0;
	cksum = IP_CSUM(mp, ulp_offset, cksum);
	*up = (uint16_t)(cksum ? cksum : ~cksum);

	return (B_TRUE);

bail:
	return (B_FALSE);
}

/*
 * Calculate the ULP checksum for IPv6. Return true if the calculation
 * was successful, or false if an error occurred. If the later, place
 * an error message into '*err'.
 */
static boolean_t
mac_sw_cksum_ipv6(mblk_t *mp, uint32_t ip_hdr_offset, const char **err)
{
	ip6_t *ip6h = (ip6_t *)(mp->b_rptr + ip_hdr_offset);
	const uint8_t proto = ip6h->ip6_nxt;
	const uint16_t *iphs = (uint16_t *)ip6h;
	/* ULP offset from start of L2. */
	uint32_t ulp_offset;
	size_t len;
	uint32_t cksum;
	uint16_t *up;
	uint16_t ip_hdr_sz;

	if (!ip_hdr_length_nexthdr_v6(mp, ip6h, &ip_hdr_sz, NULL)) {
		*err = "malformed IPv6 header";
		goto bail;
	}

	ulp_offset = ip_hdr_offset + ip_hdr_sz;

	/*
	 * We need a pointer to the ULP checksum. We're assuming the
	 * ULP checksum pointer resides in the first mblk. Our native
	 * TCP stack should always put the headers in the first mblk,
	 * but currently we have no way to guarantee that other
	 * clients don't spread headers (or even header fields) across
	 * mblks.
	 */
	switch (proto) {
	case IPPROTO_TCP:
		ASSERT3U(MBLKL(mp), >=, (ulp_offset + sizeof (tcph_t)));
		if (MBLKL(mp) < (ulp_offset + sizeof (tcph_t))) {
			*err = "mblk doesn't contain TCP header";
			goto bail;
		}

		up = IPH_TCPH_CHECKSUMP(ip6h, ip_hdr_sz);
		cksum = IP_TCP_CSUM_COMP;
		break;

	case IPPROTO_UDP:
		ASSERT3U(MBLKL(mp), >=, (ulp_offset + sizeof (udpha_t)));
		if (MBLKL(mp) < (ulp_offset + sizeof (udpha_t))) {
			*err = "mblk doesn't contain UDP header";
			goto bail;
		}

		up = IPH_UDPH_CHECKSUMP(ip6h, ip_hdr_sz);
		cksum = IP_UDP_CSUM_COMP;
		break;

	case IPPROTO_SCTP: {
		sctp_hdr_t *sctph;

		ASSERT3U(MBLKL(mp), >=, (ulp_offset + sizeof (sctp_hdr_t)));
		if (MBLKL(mp) < (ulp_offset + sizeof (sctp_hdr_t))) {
			*err = "mblk doesn't contain SCTP header";
			goto bail;
		}

		sctph = (sctp_hdr_t *)(mp->b_rptr + ulp_offset);
		/*
		 * Zero out the checksum field to ensure proper
		 * checksum calculation.
		 */
		sctph->sh_chksum = 0;
		sctph->sh_chksum = sctp_cksum(mp, ulp_offset);
		return (B_TRUE);
	}

	default:
		*err = "unexpected protocol";
		goto bail;
	}

	/*
	 * The payload length includes the payload and the IPv6
	 * extension headers; the idea is to subtract the extension
	 * header length to get the real payload length.
	 */
	len = ntohs(ip6h->ip6_plen) - (ip_hdr_sz - IPV6_HDR_LEN);
	cksum += len;

	/*
	 * We accumulate the pseudo header checksum in cksum; then we
	 * call IP_CSUM to compute the checksum over the payload.
	 */
	cksum += iphs[4] + iphs[5] + iphs[6] + iphs[7] + iphs[8] + iphs[9] +
	    iphs[10] + iphs[11] + iphs[12] + iphs[13] + iphs[14] + iphs[15] +
	    iphs[16] + iphs[17] + iphs[18] + iphs[19];
	cksum = IP_CSUM(mp, ulp_offset, cksum);

	/* For UDP/IPv6 a zero UDP checksum is not allowed. Change to 0xffff */
	if (proto == IPPROTO_UDP && cksum == 0)
		cksum = ~cksum;

	*up = (uint16_t)cksum;

	return (B_TRUE);

bail:
	return (B_FALSE);
}

/*
 * Perform software checksum on a single message, if needed. The
 * emulation performed is determined by an intersection of the mblk's
 * flags and the emul flags requested. The emul flags are documented
 * in mac.h.
 */
static mblk_t *
mac_sw_cksum(mblk_t *mp, mac_emul_t emul)
{
	mblk_t *skipped_hdr = NULL;
	uint32_t flags, start, stuff, end, value;
	uint32_t ip_hdr_offset;
	uint16_t etype;
	size_t ip_hdr_sz;
	struct ether_header *ehp;
	const char *err = "";

	/*
	 * This function should only be called from mac_hw_emul()
	 * which handles mblk chains and the shared ref case.
	 */
	ASSERT3P(mp->b_next, ==, NULL);

	mac_hcksum_get(mp, &start, &stuff, &end, &value, NULL);

	flags = DB_CKSUMFLAGS(mp);

	/* Why call this if checksum emulation isn't needed? */
	ASSERT3U(flags & (HCK_FLAGS), !=, 0);

	/*
	 * Ethernet, and optionally VLAN header. mac_hw_emul() has
	 * already verified we have enough data to read the L2 header.
	 */
	ehp = (struct ether_header *)mp->b_rptr;
	if (ntohs(ehp->ether_type) == VLAN_TPID) {
		struct ether_vlan_header *evhp;

		evhp = (struct ether_vlan_header *)mp->b_rptr;
		etype = ntohs(evhp->ether_type);
		ip_hdr_offset = sizeof (struct ether_vlan_header);
	} else {
		etype = ntohs(ehp->ether_type);
		ip_hdr_offset = sizeof (struct ether_header);
	}

	/*
	 * If this packet isn't IP, then leave it alone. We don't want
	 * to affect non-IP traffic like ARP. Assume the IP header
	 * doesn't include any options, for now. We will use the
	 * correct size later after we know there are enough bytes to
	 * at least fill out the basic header.
	 */
	switch (etype) {
	case ETHERTYPE_IP:
		ip_hdr_sz = sizeof (ipha_t);
		break;
	case ETHERTYPE_IPV6:
		ip_hdr_sz = sizeof (ip6_t);
		break;
	default:
		return (mp);
	}

	ASSERT3U(MBLKL(mp), >=, ip_hdr_offset);

	/*
	 * If the first mblk of this packet contains only the ethernet
	 * header, skip past it for now. Packets with their data
	 * contained in only a single mblk can then use the fastpaths
	 * tuned to that possibility.
	 */
	if (MBLKL(mp) == ip_hdr_offset) {
		ip_hdr_offset -= MBLKL(mp);
		/* This is guaranteed by mac_hw_emul(). */
		ASSERT3P(mp->b_cont, !=, NULL);
		skipped_hdr = mp;
		mp = mp->b_cont;
	}

	/*
	 * Both full and partial checksum rely on finding the IP
	 * header in the current mblk. Our native TCP stack honors
	 * this assumption but it's prudent to guard our future
	 * clients that might not honor this contract.
	 */
	ASSERT3U(MBLKL(mp), >=, ip_hdr_offset + ip_hdr_sz);
	if (MBLKL(mp) < (ip_hdr_offset + ip_hdr_sz)) {
		err = "mblk doesn't contain IP header";
		goto bail;
	}

	/*
	 * We are about to modify the header mblk; make sure we are
	 * modifying our own copy. The code that follows assumes that
	 * the IP/ULP headers exist in this mblk (and drops the
	 * message if they don't).
	 */
	if (DB_REF(mp) > 1) {
		mblk_t *tmp = copyb(mp);

		if (tmp == NULL) {
			err = "copyb failed";
			goto bail;
		}

		if (skipped_hdr != NULL) {
			ASSERT3P(skipped_hdr->b_cont, ==, mp);
			skipped_hdr->b_cont = tmp;
		}

		tmp->b_cont = mp->b_cont;
		freeb(mp);
		mp = tmp;
	}

	if (etype == ETHERTYPE_IP) {
		ipha_t *ipha = (ipha_t *)(mp->b_rptr + ip_hdr_offset);

		if ((flags & HCK_FULLCKSUM) && (emul & MAC_HWCKSUM_EMUL)) {
			if (!mac_sw_cksum_ipv4(mp, ip_hdr_offset, ipha, &err))
				goto bail;
		}

		/* We always update the ULP checksum flags. */
		if ((flags & HCK_FULLCKSUM) && (emul & MAC_HWCKSUM_EMULS)) {
			flags &= ~HCK_FULLCKSUM;
			flags |= HCK_FULLCKSUM_OK;
			value = 0;
		}

		/*
		 * While unlikely, it's possible to write code that
		 * might end up calling mac_sw_cksum() twice on the
		 * same mblk (performing both LSO and checksum
		 * emualtion in a single mblk chain loop -- the LSO