Source to iokit/Families/IONetworking/IOMBufMemoryCursor.cpp
/*
* Copyright (c) 1998-2000 Apple Computer, Inc. All rights reserved.
*
* @APPLE_LICENSE_HEADER_START@
*
* The contents of this file constitute Original Code as defined in and
* are subject to the Apple Public Source License Version 1.1 (the
* "License"). You may not use this file except in compliance with the
* License. Please obtain a copy of the License at
* http://www.apple.com/publicsource and read it before using this file.
*
* This Original Code and all software distributed under the License are
* distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
* EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
* INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
* FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
* License for the specific language governing rights and limitations
* under the License.
*
* @APPLE_LICENSE_HEADER_END@
*/
/* IOMBufMemoryCursor.cpp created by gvdl on 1999-1-20 */
#include <sys/cdefs.h>
__BEGIN_DECLS
#include <IOKit/assert.h>
#include <sys/param.h>
#include <sys/mbuf.h>
#include <architecture/byte_order.h>
__END_DECLS
#include <IOKit/network/IOMBufMemoryCursor.h>
#include <IOKit/IOLib.h>
#ifndef MIN
#define MIN(a,b) (((a)<(b))?(a):(b))
#endif /* MIN */
#define next_page(x) trunc_page(x + PAGE_SIZE)
/* Define the meta class stuff for the entire file here */
OSDefineMetaClass(IOMBufMemoryCursor, IOMemoryCursor)
OSDefineAbstractStructors(IOMBufMemoryCursor, IOMemoryCursor)
OSDefineMetaClassAndStructors(IOMBufNaturalMemoryCursor, IOMBufMemoryCursor)
OSDefineMetaClassAndStructors(IOMBufBigMemoryCursor, IOMBufMemoryCursor)
OSDefineMetaClassAndStructors(IOMBufLittleMemoryCursor, IOMBufMemoryCursor)
#ifdef __ppc__
OSDefineMetaClassAndStructors(IOMBufDBDMAMemoryCursor, IOMBufMemoryCursor)
#endif /* __ppc__ */
/*********************** class IOMBufMemoryCursor ***********************/
#define super IOMemoryCursor
bool IOMBufMemoryCursor::initWithSpecification(OutputSegmentFunc outSeg,
UInt maxSegmentSize,
UInt maxTransferSize,
UInt align)
{
return false;
}
bool IOMBufMemoryCursor::initWithSpecification(OutputSegmentFunc inOutSeg,
UInt inMaxSegmentSize,
UInt inMaxNumSegments)
{
if (!super::initWithSpecification(inOutSeg, inMaxSegmentSize, 0, 1))
return false;
#if 0
// It is too confusing to force the max segment size to be at least
// as large as a page. Most Enet devices only have 11-12 bit fields,
// enough for a full size frame, and also the PAGE_SIZE parameter
// may be architecture dependent.
assert(inMaxSegmentSize >= PAGE_SIZE);
if (inMaxSegmentSize < PAGE_SIZE)
return false;
#else
if (!inMaxSegmentSize)
return false;
#endif
maxSegmentSize = MIN(maxSegmentSize, PAGE_SIZE);
maxNumSegments = inMaxNumSegments;
coalesceCount = 0;
return true;
}
//
// Copy the src packet into the destination packet. The amount to copy is
// determined by the dstm->m_len, which is setup by analyseSegments, see below.
// The source mbuf is not freed nor modified.
//
#define BCOPY(s, d, l) do { bcopy((void *) s, (void *) d, l); } while(0)
static inline void coalesceSegments(struct mbuf *srcm, struct mbuf *dstm)
{
vm_offset_t src, dst;
SInt srcLen, dstLen;
struct mbuf *temp;
srcLen = srcm->m_len;
src = mtod(srcm, vm_offset_t);
dstLen = dstm->m_len;
dst = mtod(dstm, vm_offset_t);
for (;;) {
if (srcLen < dstLen) {
// Copy remainder of src mbuf to current dst.
BCOPY(src, dst, srcLen);
dst += srcLen;
dstLen -= srcLen;
// Move on to the next source mbuf.
temp = srcm->m_next; assert(temp);
srcm = temp;
srcLen = srcm->m_len;
src = mtod(srcm, vm_offset_t);
}
else if (srcLen > dstLen) {
// Copy some of src mbuf to remaining space in dst mbuf.
BCOPY(src, dst, dstLen);
src += dstLen;
srcLen -= dstLen;
// Move on to the next destination mbuf.
temp = dstm->m_next; assert(temp);
dstm = temp;
dstLen = dstm->m_len;
dst = mtod(dstm, vm_offset_t);
}
else { /* (srcLen == dstLen) */
// copy remainder of src into remaining space of current dst
BCOPY(src, dst, srcLen);
// Free current mbuf and move the current onto the next
srcm = srcm->m_next;
// Do we have any data left to copy?
if (!dstm->m_next)
break;
dstm = dstm->m_next;
assert(srcm);
dstLen = dstm->m_len;
dst = mtod(dstm, vm_offset_t);
srcLen = srcm->m_len;
src = mtod(srcm, vm_offset_t);
}
}
}
static const UInt kMBufDataCacheSize = 16;
static inline bool analyseSegments(
struct mbuf *packet, /* input packet mbuf */
const UInt mbufsInCache, /* number of entries in segsPerMBuf[] */
const UInt segsPerMBuf[], /* segments required per mbuf */
SInt numSegs, /* total number of segments */
const UInt maxSegs) /* max controller segments per mbuf */
{
struct mbuf *newPacket; // output mbuf chain.
struct mbuf *out; // current output mbuf link.
SInt outSize; // size of current output mbuf link.
SInt outSegs; // segments for current output mbuf link.
SInt doneSegs; // segments for output mbuf chain.
SInt outLen; // remaining length of input buffer.
struct mbuf *in = packet; // save the original input packet pointer.
UInt inIndex = 0;
// Allocate a mbuf (non header mbuf) to begin the output mbuf chain.
//
MGET(newPacket, M_DONTWAIT, MT_DATA);
if (!newPacket) {
IOLog("analyseSegments: MGET() 1 error\n");
return false;
}
/* Initialise outgoing packet controls */
out = newPacket;
outSize = MLEN;
doneSegs = outSegs = outLen = 0;
// numSegs stores the delta between the total and the max. For each
// input mbuf consumed, we decrement numSegs.
//
numSegs -= maxSegs;
// Loop through the input packet mbuf 'in' and construct a new mbuf chain
// large enough to make (numSegs + doneSegs + outSegs) less than or
// equal to zero.
//
do {
vm_offset_t vmo;
outLen += in->m_len;
while (outLen > outSize) {
// Oh dear the current outgoing length is too big.
if (outSize != MCLBYTES) {
// Current mbuf is not yet a cluster so promote, then
// check for error.
MCLGET(out, M_DONTWAIT);
if ( !(out->m_flags & M_EXT) ) {
IOLog("analyseSegments: MCLGET() error\n");
goto bombAnalysis;
}
outSize = MCLBYTES;
continue;
}
vmo = mtod(out, vm_offset_t);
out->m_len = MCLBYTES; /* Fill in target copy size */
doneSegs += (round_page(vmo + MCLBYTES) - trunc_page(vmo))
/ PAGE_SIZE;
// If the number of segments of the output chain, plus
// the segment for the mbuf we are about to allocate is greater
// than maxSegs, then abort.
//
if (doneSegs + 1 > (int) maxSegs) {
IOLog("analyseSegments: maxSegs limit 1 reached! %d %d\n",
doneSegs, maxSegs);
goto bombAnalysis;
}
MGET(out->m_next, M_DONTWAIT, MT_DATA);
if (!out->m_next) {
IOLog("analyseSegments: MGET() error\n");
goto bombAnalysis;
}
out = out->m_next;
outSize = MLEN;
outLen -= MCLBYTES;
}
// Compute number of segment in current outgoing mbuf.
vmo = mtod(out, vm_offset_t);
outSegs = (round_page(vmo + outLen) - trunc_page(vmo)) / PAGE_SIZE;
if (doneSegs + outSegs > (int) maxSegs) {
IOLog("analyseSegments: maxSegs limit 2 reached! %d %d %d\n",
doneSegs, outSegs, maxSegs);
goto bombAnalysis;
}
// Get the number of segments in the current inbuf
if (inIndex < mbufsInCache)
numSegs -= segsPerMBuf[inIndex]; // Yeah, in cache
else {
// Hmm, we have to recompute from scratch. Copy code from genPhys.
int thisLen = 0, mbufLen;
vmo = mtod(in, vm_offset_t);
for (mbufLen = in->m_len; mbufLen; mbufLen -= thisLen) {
thisLen = MIN(next_page(vmo), vmo + mbufLen) - vmo;
vmo += thisLen;
numSegs--;
}
}
// Walk the incoming buffer on one.
in = in->m_next;
inIndex++;
// continue looping until the total number of segments has dropped
// to an acceptable level, or if we ran out of mbuf links.
} while (in && ((numSegs + doneSegs + outSegs) > 0));
if ( (int) (numSegs + doneSegs + outSegs) <= 0) { // success
out->m_len = outLen; // Set last mbuf with the remaining length.
// The amount to copy is determine by the segment length in each
// mbuf linked to newPacket. The sum can be smaller than
// packet->pkthdr.len;
//
coalesceSegments(packet, newPacket);
// Copy complete.
// If 'in' is non zero, then it means that we only need to copy part
// of the input packet, beginning at the start. The mbuf chain
// beginning at 'in' must be preserved and linked to the new
// output packet chain. Everything before 'in', except for the
// header mbuf can be freed.
//
struct mbuf *m = packet->m_next;
while (m != in)
m = m_free(m);
// The initial header mbuf is preserved, its length set to zero, and
// linked to the new packet chain.
packet->m_len = 0;
packet->m_next = newPacket;
newPacket->m_next = in;
return true;
}
bombAnalysis:
m_freem(newPacket);
return false;
}
UInt IOMBufMemoryCursor::genPhysicalSegments(struct mbuf *packet, void *vector,
UInt maxSegs, bool doCoalesce)
{
bool doneCoalesce = false;
if (!packet || !(packet->m_flags & M_PKTHDR))
return 0;
if (!maxSegs)
maxSegs = maxNumSegments;
if (!maxSegs)
return 0;
if ( packet->m_next == 0 )
{
vm_offset_t src;
struct IOPhysicalSegment physSeg;
/*
* the packet consists of only 1 mbuf
* so if the data buffer doesn't span a page boundary
* we can take the simple way out
*/
src = mtod(packet, vm_offset_t);
if ( trunc_page(src) == trunc_page(src+packet->m_len-1) )
{
if ((physSeg.location =
(IOPhysicalAddress)mcl_to_paddr((char *)src)) == 0)
physSeg.location = (IOPhysicalAddress)pmap_extract(kernel_pmap, src);
if (!physSeg.location)
return 0;
physSeg.length = packet->m_len;
(*outSeg)(physSeg, vector, 0);
return 1;
}
}
if ( doCoalesce == true && maxSegs == 1 )
{
vm_offset_t src;
vm_offset_t dst;
struct mbuf *m;
struct mbuf *mnext;
struct mbuf *out;
UInt len = 0;
struct IOPhysicalSegment physSeg;
m = packet;
MGET(out, M_DONTWAIT, MT_DATA);
if ( out == 0 ) return 0;
MCLGET(out, M_DONTWAIT);
if ( !(out->m_flags & M_EXT) )
{
m_free( out );
return 0;
}
dst = mtod(out, vm_offset_t);
do
{
src = mtod(m, vm_offset_t);
BCOPY( src, dst, m->m_len );
dst += m->m_len;
len += m->m_len;
} while ( (m = m->m_next) != 0 );
out->m_len = len;
dst = mtod(out, vm_offset_t);
if ((physSeg.location = (IOPhysicalAddress)mcl_to_paddr((char *)dst)) == 0)
physSeg.location = (IOPhysicalAddress)pmap_extract(kernel_pmap, dst);
if (!physSeg.location)
return 0;
physSeg.length = out->m_len;
(*outSeg)(physSeg, vector, 0);
m = packet->m_next;
while (m != 0)
{
mnext = m->m_next;
m_free(m);
m = mnext;
}
// The initial header mbuf is preserved, its length set to zero, and
// linked to the new packet chain.
packet->m_len = 0;
packet->m_next = out;
out->m_next = 0;
return 1;
}
//
// Iterate over the mbuf, translating segments were allowed. When we
// are not allowed to translate segments then accumulate segment
// statistics up to kMBufDataCacheSize of mbufs. Finally
// if we overflow our cache just count how many segments this
// packet represents.
//
UInt segsPerMBuf[kMBufDataCacheSize];
tryAgain:
UInt curMBufIndex = 0;
UInt curSegIndex = 0;
UInt lastSegCount = 0;
struct mbuf *m = packet;
// For each mbuf in incoming packet.
do {
vm_size_t mbufLen, thisLen = 0;
vm_offset_t src;
// Step through each segment in the current mbuf
for (mbufLen = m->m_len, src = mtod(m, vm_offset_t);
mbufLen;
src += thisLen, mbufLen -= thisLen)
{
// If maxSegmentSize is atleast PAGE_SIZE, then
// thisLen = MIN(next_page(src), src + mbufLen) - src;
thisLen = MIN(mbufLen, maxSegmentSize);
thisLen = MIN(next_page(src), src + thisLen) - src;
// If room left then find the current segment addr and output
if (curSegIndex < maxSegs) {
struct IOPhysicalSegment physSeg;
if ((physSeg.location =
(IOPhysicalAddress)mcl_to_paddr((char *)src)) == 0)
physSeg.location = (IOPhysicalAddress)pmap_extract(kernel_pmap, src);
if (!physSeg.location)
return 0;
physSeg.length = thisLen;
(*outSeg)(physSeg, vector, curSegIndex);
}
// Count segments if we are coalescing.
curSegIndex++;
}
// Cache the segment count data if room is available.
if (curMBufIndex < kMBufDataCacheSize) {
segsPerMBuf[curMBufIndex] = curSegIndex - lastSegCount;
lastSegCount = curSegIndex;
}
// Move on to next imcoming mbuf
curMBufIndex++;
m = m->m_next;
} while (m);
// If we finished cleanly return number of segments found
if (curSegIndex <= maxSegs)
return curSegIndex;
if (!doCoalesce)
return 0; // if !coalescing we've got a problem.
// If we are coalescing and it is possible then attempt coalesce,
if (!doneCoalesce
&& (UInt) packet->m_pkthdr.len <= maxSegs * maxSegmentSize) {
// Hmm, we have to do some coalescing.
bool analysisRet;
analysisRet = analyseSegments(packet,
MIN(curMBufIndex, kMBufDataCacheSize),
segsPerMBuf,
curSegIndex, maxSegs);
if (analysisRet) {
doneCoalesce = true;
coalesceCount++;
goto tryAgain;
}
}
assert(!doneCoalesce); // Problem in Coalesce code.
packetTooBigErrors++;
return 0;
}
/********************* class IOMBufBigMemoryCursor **********************/
IOMBufBigMemoryCursor *
IOMBufBigMemoryCursor::withSpecification(UInt maxSegSize, UInt maxNumSegs)
{
IOMBufBigMemoryCursor *me = new IOMBufBigMemoryCursor;
if (me && !me->initWithSpecification(&bigOutputSegment,
maxSegSize, maxNumSegs)) {
me->release();
return 0;
}
return me;
}
/******************* class IOMBufNaturalMemoryCursor ********************/
IOMBufNaturalMemoryCursor *
IOMBufNaturalMemoryCursor::withSpecification(UInt maxSegSize, UInt maxNumSegs)
{
IOMBufNaturalMemoryCursor *me = new IOMBufNaturalMemoryCursor;
if (me && !me->initWithSpecification(&naturalOutputSegment,
maxSegSize, maxNumSegs)) {
me->release();
return 0;
}
return me;
}
/******************** class IOMBufLittleMemoryCursor ********************/
IOMBufLittleMemoryCursor *
IOMBufLittleMemoryCursor::withSpecification(UInt maxSegSize, UInt maxNumSegs)
{
IOMBufLittleMemoryCursor *me = new IOMBufLittleMemoryCursor;
if (me && !me->initWithSpecification(&littleOutputSegment,
maxSegSize, maxNumSegs)) {
me->release();
return 0;
}
return me;
}
/******************** class IOMBufDBDMAMemoryCursor *********************/
#ifdef __ppc__
IOMBufDBDMAMemoryCursor *
IOMBufDBDMAMemoryCursor::withSpecification(UInt maxSegSize, UInt maxNumSegs)
{
IOMBufDBDMAMemoryCursor *me = new IOMBufDBDMAMemoryCursor;
if (me && !me->initWithSpecification(&dbdmaOutputSegment,
maxSegSize, maxNumSegs)) {
me->release();
return 0;
}
return me;
}
#endif /* __ppc__ */