|
|
1.1 ! root 1: /* ! 2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved. ! 3: * ! 4: * @APPLE_LICENSE_HEADER_START@ ! 5: * ! 6: * The contents of this file constitute Original Code as defined in and ! 7: * are subject to the Apple Public Source License Version 1.1 (the ! 8: * "License"). You may not use this file except in compliance with the ! 9: * License. Please obtain a copy of the License at ! 10: * http://www.apple.com/publicsource and read it before using this file. ! 11: * ! 12: * This Original Code and all software distributed under the License are ! 13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER ! 14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES, ! 15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY, ! 16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the ! 17: * License for the specific language governing rights and limitations ! 18: * under the License. ! 19: * ! 20: * @APPLE_LICENSE_HEADER_END@ ! 21: */ ! 22: /* ! 23: * Copyright (c) 1990, 1995-1998 Apple Computer, Inc. ! 24: * All Rights Reserved. ! 25: */ ! 26: ! 27: /* Control.c ! 28: * From Mike Shoemaker v01.25 07/02/90 for MacOS ! 29: * 09/07/95 - Modified for performance (Tuyen Nguyen) ! 30: * Modified for MP, 1996 by Tuyen Nguyen ! 31: * Modified, April 9, 1997 by Tuyen Nguyen for MacOSX. ! 32: */ ! 33: ! 34: #include <sys/errno.h> ! 35: #include <sys/types.h> ! 36: #include <sys/param.h> ! 37: #include <machine/spl.h> ! 38: #include <sys/systm.h> ! 39: #include <sys/kernel.h> ! 40: #include <sys/proc.h> ! 41: #include <sys/filedesc.h> ! 42: #include <sys/fcntl.h> ! 43: #include <sys/mbuf.h> ! 44: #include <sys/time.h> ! 45: #include <sys/socket.h> ! 46: ! 47: #include <netat/sysglue.h> ! 48: #include <netat/appletalk.h> ! 49: #include <netat/ddp.h> ! 50: #include <netat/at_pcb.h> ! 51: #include <netat/debug.h> ! 52: #include <netat/adsp.h> ! 53: #include <netat/adsp_internal.h> ! 54: ! 55: /* # of additional ticks to add to any timer that we're queuing up. For ! 56: * very short delays (1 and 2), the timer fires before the transmit ! 57: * even takes place */ ! 58: #define TX_DLY 2 ! 59: ! 60: int adsp_window = 1; ! 61: ! 62: /* ! 63: * CalcRecvWdw ! 64: * ! 65: * INPUTS: ! 66: * sp ADSP Stream ! 67: * OUTPUTS: ! 68: * # of bytes in avail in local receive queue ! 69: */ ! 70: int CalcRecvWdw(sp) /* (CCBPtr sp) */ ! 71: CCBPtr sp; ! 72: { ! 73: int bytes; ! 74: ! 75: bytes = calcRecvQ(sp); ! 76: bytes = sp->rbuflen - bytes; /* get what is left */ ! 77: ! 78: if (bytes <= 16) { /* %%% this should be zero */ ! 79: sp->rbufFull = 1; /* Save flag that our recv buf is full */ ! 80: return 0; ! 81: } ! 82: else ! 83: return ((bytes+bytes+bytes) >> 2) + 1; /* %%% */ ! 84: } ! 85: ! 86: calcRecvQ(sp) ! 87: CCBPtr sp; ! 88: { ! 89: int bytes = 0; ! 90: #ifdef AT_Socket ! 91: register struct mbuf *m, *p; ! 92: ! 93: if (((sp->gref)->so)->so_rcv.sb_mb) ! 94: for (p = ((sp->gref)->so)->so_rcv.sb_mb; p; p = p->m_nextpkt) ! 95: for (m = p; m; m = m->m_next) ! 96: bytes += m->m_len; ! 97: #else ! 98: register gbuf_t *mb; ! 99: ! 100: if (sp->rData) { /* There is data in buffer */ ! 101: if (mb = sp->rbuf_mb) { ! 102: do { ! 103: bytes += gbuf_msgsize(mb); ! 104: mb = gbuf_next(mb); ! 105: } while (mb); ! 106: } ! 107: if (mb = sp->crbuf_mb) ! 108: bytes += gbuf_msgsize(mb); ! 109: } ! 110: #endif ! 111: return bytes; ! 112: } ! 113: ! 114: /* ! 115: * CheckSend ! 116: * ! 117: * Check to see if the transmit PB is available and if there is anything ! 118: * to transmit. Start off any pending transmit. ! 119: * ! 120: * Normally called from the write completion routine ! 121: * ! 122: * INPUTS: ! 123: * sp Connection control block ! 124: * OUTPUTS: ! 125: * true if sent a packet ! 126: */ ! 127: void CheckSend(sp) /* (CCBPtr sp) */ ! 128: register CCBPtr sp; ! 129: { ! 130: int i; ! 131: int attnMsg; /* True if attention message */ ! 132: int s; ! 133: register gbuf_t *mp; /* send message block */ ! 134: #ifdef notdef ! 135: register gbuf_t *tmp; ! 136: u_char current; ! 137: #endif ! 138: char *dp; /* a data pointer */ ! 139: int use_attention_code; ! 140: int len; /* length used in allocd mblk */ ! 141: int datalen; /* amount of data attached to mblk */ ! 142: gbuf_t *mprev, *mlist = 0; ! 143: ! 144: top: ! 145: ! 146: if (sp->state == sClosed) ! 147: return; ! 148: ! 149: /* get a message block to hold DDP and ! 150: * ADSP headers + 2 bytes of attention ! 151: * code if necessary */ ! 152: if ((mp = gbuf_alloc(AT_WR_OFFSET + DDPL_FRAME_LEN + ADSP_FRAME_LEN + ADSP_OPEN_FRAME_LEN + 2, ! 153: PRI_LO)) == 0) { ! 154: if (mlist) ! 155: gbuf_freel(mlist); ! 156: return; /* can't get buffers... do nothing! */ ! 157: } ! 158: ATDISABLE(s, sp->lock); ! 159: sp->callSend = 0; /* Clear flag */ ! 160: use_attention_code = 0; ! 161: len = 0; ! 162: datalen = 0; ! 163: ! 164: gbuf_rinc(mp,AT_WR_OFFSET); ! 165: gbuf_wset(mp,DDPL_FRAME_LEN); /* leave room for DDP header */ ! 166: ! 167: if (sp->sendCtl) { ! 168: short mask; ! 169: ! 170: i = sp->sendCtl; /* get local copy bitmap of */ ! 171: /* which ctl packets to send. */ ! 172: attnMsg = 0; ! 173: ! 174: if (i & 0x1E) /* One of the open ctrl packets */ ! 175: { ! 176: ! 177: /* point past ADSP header (no attention) */ ! 178: dp = ((char *) gbuf_wptr(mp)) + ADSP_FRAME_LEN; ! 179: UAL_ASSIGN(sp->f.pktFirstByteSeq, netdw(sp->firstRtmtSeq)); ! 180: ! 181: UAS_ASSIGN(sp->of.version, netw(0x0100)); /* Fill in open connection parms */ ! 182: UAS_ASSIGN(sp->of.dstCID, sp->remCID); /* Destination CID */ ! 183: UAL_ASSIGN(sp->of.pktAttnRecvSeq, netdw(sp->attnRecvSeq)); ! 184: bcopy((caddr_t) &sp->of, (caddr_t) dp, ADSP_OPEN_FRAME_LEN); ! 185: len += ADSP_OPEN_FRAME_LEN; ! 186: ! 187: if (i & B_CTL_OREQ) { ! 188: UAS_ASSIGN(sp->f.CID, sp->locCID); ! 189: mask = B_CTL_OREQ; ! 190: sp->f.descriptor = ADSP_CONTROL_BIT | ADSP_CTL_OREQ; ! 191: } else if (i & B_CTL_OACK) { ! 192: UAS_ASSIGN(sp->f.CID, sp->locCID); ! 193: mask = B_CTL_OACK; ! 194: sp->f.descriptor = ADSP_CONTROL_BIT | ADSP_CTL_OACK; ! 195: } else if (i & B_CTL_OREQACK) { ! 196: UAS_ASSIGN(sp->f.CID, sp->locCID); ! 197: mask = B_CTL_OREQACK; ! 198: sp->f.descriptor = ADSP_CONTROL_BIT | ADSP_CTL_OREQACK; ! 199: } else /* Deny */ ! 200: { ! 201: UAS_ASSIGN(sp->f.CID, 0); ! 202: mask = B_CTL_ODENY; ! 203: sp->f.descriptor = ADSP_CONTROL_BIT | ADSP_CTL_ODENY; ! 204: UAL_ASSIGN(sp->f.pktFirstByteSeq, 0); ! 205: } ! 206: ! 207: if (i & (B_CTL_OREQ | B_CTL_OREQACK)) ! 208: /* Need to start up a timer for it */ ! 209: { ! 210: /* It's possible that we've received a duplicate ! 211: * open request. In this case, there will already be ! 212: * a timer queued up for the request+ack ! 213: * packet we sent the first time. So remove the timer ! 214: * and start another. ! 215: */ ! 216: RemoveTimerElem(&adspGlobal.slowTimers, &sp->ProbeTimer); ! 217: InsertTimerElem(&adspGlobal.slowTimers, &sp->ProbeTimer, ! 218: sp->openInterval+1); ! 219: } ! 220: } else { ! 221: /* seq # of next byte to send */ ! 222: UAL_ASSIGN(sp->f.pktFirstByteSeq, netdw(sp->sendSeq)); ! 223: ! 224: if (i & B_CTL_CLOSE) { ! 225: sp->state = sClosed; /* Now we're closed */ ! 226: mask = B_CTL_CLOSE; ! 227: sp->f.descriptor = ADSP_CONTROL_BIT | ADSP_CTL_CLOSE; ! 228: } else if (i & B_CTL_PROBE) { ! 229: mask = B_CTL_PROBE; ! 230: sp->f.descriptor = ! 231: ADSP_CONTROL_BIT | ADSP_CTL_PROBE | ADSP_ACK_REQ_BIT; ! 232: } else if (i & B_CTL_FRESET) { ! 233: mask = B_CTL_FRESET; ! 234: sp->f.descriptor = ADSP_CONTROL_BIT | ADSP_CTL_FRESET; ! 235: InsertTimerElem(&adspGlobal.fastTimers, ! 236: &sp->ResetTimer, sp->rtmtInterval+TX_DLY); ! 237: } else if (i & B_CTL_FRESETACK) { ! 238: mask = B_CTL_FRESETACK; ! 239: sp->f.descriptor = ADSP_CONTROL_BIT | ADSP_CTL_FRESET_ACK; ! 240: } ! 241: else if (i & B_CTL_RETRANSMIT) { ! 242: mask = B_CTL_RETRANSMIT; ! 243: sp->f.descriptor = ADSP_CONTROL_BIT | ADSP_CTL_RETRANSMIT; ! 244: } ! 245: else { ! 246: dPrintf(D_M_ADSP, D_L_ERROR, ("CheckSend: Control bit error\n")); ! 247: } ! 248: } /* non open control packet */ ! 249: ! 250: sp->sendCtl &= ~mask; ! 251: goto sendit; ! 252: } /* send control packet */ ! 253: ! 254: if (sp->sendAttnData) /* Send attn ready to go? */ ! 255: { ! 256: sp->sendAttnData = 0; /* Clear Flags */ ! 257: if (sp->sapb) { ! 258: sp->sendAttnAck = 0; /* This will also do an Attn Ack */ ! 259: ! 260: attnMsg = 1; ! 261: sp->f.descriptor = ADSP_ATTENTION_BIT | ADSP_ACK_REQ_BIT; ! 262: if (gbuf_cont(sp->sapb->mp)) { ! 263: gbuf_cont(mp) = gbuf_dupm(gbuf_cont(sp->sapb->mp)); ! 264: /* Major hack here. The ADSP Attn code is butted up against ! 265: * the end of the adsp packet header, and the length is ! 266: * increased by 2. (There is a pad field behind the adsp ! 267: * header in the CCB just for this purpose.) ! 268: */ ! 269: } ! 270: use_attention_code++; ! 271: ! 272: sp->f.data[0] = high(sp->sapb->u.attnParams.attnCode); ! 273: sp->f.data[1] = low(sp->sapb->u.attnParams.attnCode); ! 274: InsertTimerElem(&adspGlobal.fastTimers, &sp->AttnTimer, ! 275: sp->rtmtInterval+TX_DLY); ! 276: goto sendit; ! 277: } ! 278: } /* attn data */ ! 279: ! 280: if (sp->sendAttnAck) /* Send attn ack ready to go? */ ! 281: { ! 282: attnMsg = 1; ! 283: sp->f.descriptor = ADSP_CONTROL_BIT | ADSP_ATTENTION_BIT; ! 284: sp->sendAttnAck = 0; ! 285: goto sendit; ! 286: } /* attn ack */ ! 287: ! 288: if ((sp->state == sOpen || sp->state == sClosing) && /* Correct state */ ! 289: (!sp->waitingAck) && /* not waiting for an ACK */ ! 290: (sp->sData) && /* have data to send */ ! 291: (GTE(sp->sendWdwSeq,sp->sendSeq)) && /* he has room to accept it */ ! 292: (sp->pktSendCnt < sp->pktSendMax)) /* haven't sent too many pkts ! 293: * in a row. */ ! 294: { ! 295: attnMsg = 0; ! 296: if (datalen = attachData(sp, mp)) /* attach data to mp */ ! 297: goto sendit; /* if successful, sendit */ ! 298: } ! 299: ! 300: if (sp->sendDataAck) { ! 301: UAL_ASSIGN(sp->f.pktFirstByteSeq, netdw(sp->sendSeq)); /* seq # of next byte */ ! 302: attnMsg = 0; ! 303: sp->f.descriptor = ADSP_CONTROL_BIT; ! 304: goto sendit; ! 305: } ! 306: ! 307: /* ! 308: * Nothing left to do... ! 309: */ ! 310: if (mp) ! 311: gbuf_freem(mp); ! 312: ATENABLE(s, sp->lock); ! 313: if (mlist) ! 314: adsp_sendddp(sp, mlist, 0, &sp->remoteAddress, DDP_ADSP); ! 315: return; ! 316: ! 317: sendit: ! 318: ! 319: if (attnMsg) { ! 320: UAL_ASSIGN(sp->f.pktFirstByteSeq, netdw(sp->attnSendSeq)); ! 321: UAL_ASSIGN(sp->f.pktNextRecvSeq, netdw(sp->attnRecvSeq)); ! 322: UAS_ASSIGN(sp->f.pktRecvWdw, 0); /* Always zero in attn pkt */ ! 323: } else { ! 324: sp->sendDataAck = 0; ! 325: UAL_ASSIGN(sp->f.pktNextRecvSeq, netdw(sp->recvSeq)); ! 326: UAS_ASSIGN(sp->f.pktRecvWdw, netw(CalcRecvWdw(sp))); ! 327: } ! 328: if (use_attention_code) { ! 329: bcopy((caddr_t) &sp->f, (caddr_t) gbuf_wptr(mp), ADSP_FRAME_LEN + 2); ! 330: len += ADSP_FRAME_LEN + 2; ! 331: } else { ! 332: bcopy((caddr_t) &sp->f, (caddr_t) gbuf_wptr(mp), ADSP_FRAME_LEN); ! 333: len += ADSP_FRAME_LEN; ! 334: } ! 335: gbuf_winc(mp,len); /* update mblk length */ ! 336: if (mlist) ! 337: gbuf_next(mprev) = mp; ! 338: else ! 339: mlist = mp; ! 340: mprev = mp; ! 341: ! 342: if (sp->state == sClosed) { /* must have sent a close advice */ ! 343: /* send header + data */ ! 344: ATENABLE(s, sp->lock); ! 345: adsp_sendddp(sp, mlist, 0, &sp->remoteAddress, DDP_ADSP); ! 346: DoClose(sp, 0, -1); /* complete close! */ ! 347: return; ! 348: } ! 349: ATENABLE(s, sp->lock); ! 350: if (sp->state == sClosing) /* See if we were waiting on this write */ ! 351: CheckOkToClose(sp); ! 352: goto top; ! 353: } ! 354: ! 355: /* ! 356: * completepb delivers a paramater block with all its appropriate fields ! 357: * set back to the user. ! 358: * ! 359: * The assumptions here are that the PB is not linked to any queue, ! 360: * that the fields including ioResult are set, and that the ! 361: * kernel is no longer interested in the mblks that may or ! 362: * maynot be linked to this pb. ! 363: */ ! 364: void completepb(sp, pb) ! 365: register CCBPtr sp; ! 366: register struct adspcmd *pb; ! 367: { ! 368: if (sp->gref && (sp->gref->info == (caddr_t)sp->sp_mp)) { ! 369: if (gbuf_len(pb->mp) > sizeof(struct adspcmd)) ! 370: gbuf_wset(pb->mp,sizeof(struct adspcmd)); ! 371: SndMsgUp(sp->gref, pb->mp); ! 372: NotifyUser(sp); ! 373: } else ! 374: gbuf_freem(pb->mp); ! 375: } ! 376: ! 377: attachData(sp, mp) ! 378: register CCBPtr sp; ! 379: register gbuf_t *mp; ! 380: { ! 381: int seq; ! 382: int cnt; ! 383: char eom = 0; ! 384: int bsize; ! 385: int diff; ! 386: char sendAckReq; ! 387: int partial = 0; /* flag for a partial send */ ! 388: int tcnt = 0; ! 389: register gbuf_t *smp; /* send data message block */ ! 390: register gbuf_t *psmp; /* previous message block */ ! 391: ! 392: sendAckReq = 0; ! 393: ! 394: if (LT(sp->sendSeq, sp->firstRtmtSeq)) /* Sanity check on send seq */ ! 395: sp->sendSeq = sp->firstRtmtSeq; /* seq must be oldest in buffer. */ ! 396: ! 397: /* This test and assignment was necessary because the retry VBL could ! 398: * have fired and reset send Seq to first Rtmt Seq, and then an ! 399: * expected ACK comes in that bumps first Rtmt Seq up. Then we ! 400: * have the problem that send Seq is less than first Rtmt Seq. ! 401: * The easiest fix to this timing dilemma seems to be to reset ! 402: * sendSeq to first Rtmt Seq if we're sending the first packet. ! 403: */ ! 404: UAL_ASSIGN(sp->f.pktFirstByteSeq, netdw(sp->sendSeq)); ! 405: ! 406: if (smp = sp->sbuf_mb) /* Get oldest header */ ! 407: eom = 1; ! 408: else if (smp = sp->csbuf_mb) ! 409: eom = 0; ! 410: ! 411: if (smp == 0) { /* this shouldn't happen... */ ! 412: sp->sData = 0; ! 413: return 0; ! 414: } ! 415: /* ! 416: * Must find next byte to transmit ! 417: */ ! 418: seq = sp->firstRtmtSeq; /* Seq # of oldest in buffer */ ! 419: while ((diff = (sp->sendSeq - seq)) >= ((bsize = gbuf_msgsize(smp)) + eom)) { ! 420: seq += bsize + eom; /* update sequence # */ ! 421: if (gbuf_next(smp)) { /* if another send buffer */ ! 422: smp = gbuf_next(smp); ! 423: eom = 1; ! 424: } else if (smp == sp->csbuf_mb) { /* seen the current one? */ ! 425: smp = 0; ! 426: break; ! 427: } else if (sp->csbuf_mb) { /* look at it */ ! 428: smp = sp->csbuf_mb; ! 429: eom = 0; ! 430: } else { /* no more buffers */ ! 431: smp = 0; ! 432: break; ! 433: } ! 434: } /* while */ ! 435: ! 436: if (smp) { ! 437: if (gbuf_next(smp) == 0) /* last block */ ! 438: sendAckReq = 1; ! 439: cnt = bsize - diff; /* # of bytes in this block */ ! 440: } else ! 441: cnt = 0; ! 442: ! 443: /* ! 444: * Check to see if the number of bytes is less than the 'send ! 445: * Blocking' setting. If so, then we won't send this data unless ! 446: * we're flushing. So we set up a timer to force a flush later. ! 447: */ ! 448: if ((cnt < sp->sendBlocking) && !sp->writeFlush) { ! 449: InsertTimerElem(&adspGlobal.fastTimers, &sp->FlushTimer, ! 450: sp->sendInterval); ! 451: return 0; /* no data to send */ ! 452: } ! 453: ! 454: if (cnt > ADSP_MAX_DATA_LEN) { /* truncate to one packet */ ! 455: cnt = ADSP_MAX_DATA_LEN; ! 456: eom = 0; ! 457: sendAckReq = 0; /* Won't send ack because end of data */ ! 458: partial++; ! 459: } ! 460: ! 461: if (smp) { ! 462: /* trim extra bytes off the beginning of the "block" before the copy */ ! 463: while (diff) { ! 464: if (gbuf_len(smp) > diff) ! 465: break; ! 466: else ! 467: diff -= gbuf_len(smp); ! 468: smp = gbuf_cont(smp); ! 469: } ! 470: if((gbuf_cont(mp) = gbuf_dupm(smp)) == 0) /* copy the data */ ! 471: return 0; ! 472: smp = gbuf_cont(mp); /* use the new message blocks */ ! 473: gbuf_rinc(smp,diff); /* and get to the first byte of data to send */ ! 474: } ! 475: /* ! 476: * Check to see if this many bytes will close the other end's ! 477: * receive window. If so, we need to send an ack request along ! 478: * with this. sendWdwSeq is the seq # of the last byte that ! 479: * the remote has room for ! 480: */ ! 481: if ((diff = sp->sendWdwSeq + 1 - sp->sendSeq) <= cnt) { ! 482: if (diff < cnt) { /* Won't fit exactly */ ! 483: eom = 0; /* so can't send EOM */ ! 484: cnt = diff; ! 485: partial++; ! 486: } ! 487: sendAckReq = 1; /* Make him tell us new recv. window */ ! 488: sp->noXmitFlow = 1; /* Don't do flow control calc. */ ! 489: } ! 490: ! 491: /* trim extra bytes off the tail of the "block" after the copy */ ! 492: if (partial && smp) { ! 493: psmp = smp; ! 494: tcnt = cnt; ! 495: while (tcnt && smp) { /* while there are message blocks and data */ ! 496: if (tcnt >= gbuf_len(smp)) { ! 497: tcnt -= gbuf_len(smp); ! 498: if (tcnt) { ! 499: psmp = smp; ! 500: smp = gbuf_cont(smp); ! 501: } else { ! 502: if (psmp != smp) { /* not the first item on the list */ ! 503: gbuf_cont(psmp) = 0; ! 504: gbuf_freem(smp); ! 505: smp = psmp; ! 506: } else { ! 507: gbuf_freem(gbuf_cont(smp)); ! 508: gbuf_cont(smp) = 0; ! 509: } ! 510: break; ! 511: } ! 512: } else { ! 513: gbuf_wset(smp,tcnt); ! 514: if (gbuf_cont(smp)) { ! 515: gbuf_freem(gbuf_cont(smp)); ! 516: gbuf_cont(smp) = 0; ! 517: } ! 518: break; ! 519: } ! 520: } ! 521: } ! 522: ! 523: sp->sendSeq += cnt + eom; /* Update sendSeq field */ ! 524: ! 525: if (GT(sp->sendSeq, sp->maxSendSeq)) /* Keep track of >st ever sent */ ! 526: sp->maxSendSeq = sp->sendSeq; ! 527: ! 528: if (eom) ! 529: sp->f.descriptor = ADSP_EOM_BIT; ! 530: else ! 531: sp->f.descriptor = 0; ! 532: ! 533: if (sendAckReq || (++sp->pktSendCnt >= sp->pktSendMax)) { ! 534: /* Last packet in a series */ ! 535: sp->f.descriptor |= ADSP_ACK_REQ_BIT; /* We want an ack to this */ ! 536: sp->waitingAck = 1; /* Flag that we're waiting */ ! 537: sp->sendStamp = SysTicks(); /* Save time we sent request */ ! 538: sp->timerSeq = sp->sendSeq; /* Save seq # we want acked */ ! 539: InsertTimerElem(&adspGlobal.fastTimers, &sp->RetryTimer, ! 540: sp->rtmtInterval+TX_DLY); ! 541: } ! 542: return cnt + eom; ! 543: } ! 544: ! 545: ! 546:
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.