|
|
1.1 ! root 1: /* ! 2: * tcp_device.c ! 3: */ ! 4: ! 5: #include "sys/param.h" ! 6: #include "sys/stream.h" ! 7: #include "sys/inio.h" ! 8: #include "sys/ttyio.h" ! 9: #include "sys/conf.h" ! 10: #include "sys/user.h" ! 11: #include "sys/inet/in.h" ! 12: #include "sys/inet/ip_var.h" ! 13: #include "sys/inet/tcp.h" ! 14: #include "sys/inet/tcp_timer.h" ! 15: #include "sys/inet/tcp_seq.h" ! 16: #include "sys/inet/tcp_var.h" ! 17: #include "sys/inet/tcpip.h" ! 18: #include "sys/inet/tcp_user.h" ! 19: #include "sys/inet/tcp_fsm.h" ! 20: ! 21: extern int tcp_busy; /* set to discourage timers & ensuing panics */ ! 22: extern int tcp_maxseg; ! 23: ! 24: long tcpdopen(); ! 25: int tcpdclose(), tcpdput(); ! 26: int tcpdosrv(), tcpdisrv(); ! 27: static struct qinit tcpdrinit = { noput, tcpdisrv, tcpdopen, tcpdclose, 2048, 64 }; ! 28: struct qinit tcpdwinit = { tcpdput, tcpdosrv, tcpdopen, tcpdclose, 2048, 64}; ! 29: struct streamtab tcpdinfo = { &tcpdrinit, &tcpdwinit }; ! 30: ! 31: struct cdevsw tcpcdev = cstrinit(&tcpdinfo); ! 32: ! 33: extern int tcpcnt; ! 34: extern struct tcpcb tcpcb[]; ! 35: ! 36: long ! 37: tcpdopen(q, dev) ! 38: register struct queue *q; ! 39: dev_t dev; ! 40: { ! 41: struct tcpcb *tp; ! 42: ! 43: dev = minor(dev); ! 44: if(dev >= tcpcnt) ! 45: return(0); ! 46: tp = &tcpcb[dev]; ! 47: if(dev&01){ ! 48: /* outgoing channel */ ! 49: /* ! 50: * disallow if active ! 51: */ ! 52: if(tp->t_state != TCPS_CLOSED) ! 53: return(0); ! 54: /* ! 55: * only one opener at a time for outgoing channels ! 56: */ ! 57: if(q->ptr) ! 58: return(0); ! 59: } else { ! 60: /* incoming channel */ ! 61: /* ! 62: * disallow if inactive ! 63: */ ! 64: if(tp->t_state == TCPS_CLOSED) { ! 65: printf("tcp%d closed\n", dev); ! 66: return(0); ! 67: } ! 68: /* ! 69: * disallow if it's not open or waiting to be opened ! 70: */ ! 71: if((tp->so_state&(SS_OPEN|SS_PLEASEOPEN))==0) { ! 72: printf("tcp%d state %x\n", dev, tp->so_state); ! 73: return(0); ! 74: } ! 75: /* ! 76: * if already open, just accept ! 77: */ ! 78: if(q->ptr) ! 79: return(1); ! 80: } ! 81: tcp_busy++; ! 82: if((tp->so_state & SS_PLEASEOPEN) == 0){ ! 83: tp->so_rcount = tp->so_wcount = tp->so_options = 0; ! 84: tp->so_laddr = tp->so_faddr = (in_addr)0; ! 85: tp->so_lport = tp->so_fport = (tcp_port)0; ! 86: tp->so_oobmark = tp->so_delimcnt = 0; ! 87: tp->so_head = (struct tcpcb *)0; ! 88: tp->so_state = SS_WAITING; ! 89: tp->t_maxseg = TCP_DEFMAXSEG; /* until changed by other end */ ! 90: } ! 91: tp->so_state |= SS_OPEN; ! 92: tp->so_dev = dev; ! 93: tp->so_rq = q; ! 94: tp->so_wq = WR(q); ! 95: q->ptr = (caddr_t)tp; ! 96: WR(q)->flag |= QBIGB; ! 97: WR(q)->ptr = (caddr_t)tp; ! 98: --tcp_busy; ! 99: if(tp->so_state & SS_PLEASEOPEN){ ! 100: tp->so_state &= ~SS_PLEASEOPEN; ! 101: qenable(WR(q)); /* to force out rcv wnd update */ ! 102: } ! 103: return(1); ! 104: } ! 105: ! 106: tcpdclose(q) ! 107: register struct queue *q; ! 108: { ! 109: struct tcpcb *tp; ! 110: ! 111: tp = (struct tcpcb *)q->ptr; ! 112: tcp_busy++; ! 113: if(tp == 0){ ! 114: --tcp_busy; ! 115: return; ! 116: } ! 117: tp->so_state &= ~(SS_OPEN|SS_WAITING); ! 118: tp->so_rq = tp->so_wq = (struct queue *) 0; ! 119: tp->so_wcount = tp->so_rcount = 0; ! 120: if(tp->t_state > TCPS_LISTEN) ! 121: tcp_disconnect(tp); ! 122: else ! 123: tcp_close(tp); ! 124: --tcp_busy; ! 125: } ! 126: ! 127: tcpdput(q, bp) ! 128: register struct queue *q; ! 129: register struct block *bp; ! 130: { ! 131: int s, x; ! 132: struct tcpcb *tp; ! 133: struct block *nbp; ! 134: struct foo { ! 135: int com; ! 136: struct tcpuser rep; ! 137: }; ! 138: struct foo *fp; ! 139: ! 140: tp = (struct tcpcb *)q->ptr; ! 141: switch(bp->type){ ! 142: case M_IOCTL: ! 143: bp->type = M_IOCACK; ! 144: switch(stiocom(bp)){ ! 145: case TIOCSETP: ! 146: case TIOCSETN: ! 147: x = ((struct ttydevb *)stiodata(bp))->ispeed; ! 148: bp->wptr = bp->rptr; ! 149: bp->type = M_IOCACK; ! 150: qreply(q, bp); ! 151: if(x == 0) ! 152: qpctl(OTHERQ(q), M_HANGUP); ! 153: return; ! 154: case TIOCGETP: ! 155: ((struct ttydevb *)stiodata(bp))->ispeed = ! 156: ((struct ttydevb *)stiodata(bp))->ospeed = B9600; ! 157: break; ! 158: case TCPGETADDR: ! 159: nbp = allocb(sizeof(struct foo)); ! 160: if (nbp == NULL) { ! 161: bp->type = M_IOCNAK; ! 162: break; ! 163: } ! 164: nbp->type = M_IOCACK; ! 165: fp = (struct foo *)nbp->rptr; ! 166: fp->com = stiocom(bp); ! 167: fp->rep.lport = tp->so_lport; ! 168: fp->rep.fport = tp->so_fport; ! 169: fp->rep.laddr = tp->so_laddr; ! 170: fp->rep.faddr = tp->so_faddr; ! 171: nbp->wptr = nbp->rptr + sizeof(struct foo); ! 172: freeb(bp); ! 173: bp = nbp; ! 174: break; ! 175: case TCPIOHUP: ! 176: tp->so_state |= SS_HANGUP; ! 177: bp->wptr = bp->rptr; ! 178: bp->type = M_IOCACK; ! 179: qreply(q, bp); ! 180: return; ! 181: default: ! 182: bp->type = M_IOCNAK; ! 183: } ! 184: qreply(q, bp); ! 185: return; ! 186: case M_DATA: ! 187: if(socantsendmore(tp)){ ! 188: freeb(bp); ! 189: return; ! 190: } ! 191: s = spl6(); ! 192: if (bp->wptr!=bp->rptr) ! 193: tp->so_delimcnt = 0; ! 194: if (bp->class&S_DELIM) ! 195: tp->so_delimcnt++; /* 2 delims are logical EOF */ ! 196: if (bp->wptr==bp->rptr){ ! 197: freeb(bp); ! 198: } else { ! 199: tp->so_wcount += BLEN(bp); /* BEFORE the putq */ ! 200: putq(q, bp); ! 201: } ! 202: splx(s); ! 203: if(tp->so_delimcnt ! 204: || (tp->so_options&(SO_ACCEPTCONN|SS_WAITING)) == 0) ! 205: qenable(q); ! 206: break; ! 207: default: ! 208: freeb(bp); ! 209: break; ! 210: } ! 211: } ! 212: ! 213: tcpdosrv(q) ! 214: struct queue *q; ! 215: { ! 216: register struct tcpcb *tp; ! 217: ! 218: if((tp = (struct tcpcb *)q->ptr) == 0) ! 219: return; ! 220: if (tp->so_state&SS_WCLOSED) ! 221: return; ! 222: tcp_busy++; ! 223: if(tp->so_delimcnt > 1){ ! 224: tp->so_state |= SS_WCLOSED; ! 225: tp = tcp_usrclosed(tp); ! 226: if(tp) ! 227: tcp_output(tp); ! 228: } else if ((tp->so_options & SO_ACCEPTCONN) == 0 ! 229: && (tp->so_state & SS_WAITING) == 0) ! 230: tcp_output(tp); ! 231: else ! 232: tcpduser(tp); ! 233: --tcp_busy; ! 234: } ! 235: ! 236: tcpdrint(bp, tp) ! 237: register struct block *bp; ! 238: register struct tcpcb *tp; ! 239: { ! 240: register struct block *bp1; ! 241: ! 242: if (tp->so_rq == NULL) { ! 243: printf("tcpdrint but no tp->so_rq\n"); ! 244: bp_free(bp); ! 245: return -1; ! 246: } ! 247: while(bp){ ! 248: bp1 = bp->next; ! 249: tp->so_rcount += bp->wptr - bp->rptr; ! 250: if(bp->wptr == bp->rptr) ! 251: freeb(bp); ! 252: else ! 253: putq(tp->so_rq, bp); ! 254: bp = bp1; ! 255: } ! 256: return 0; ! 257: } ! 258: ! 259: tcpdisrv(q) ! 260: register struct queue *q; ! 261: { ! 262: register struct tcpcb *tp = (struct tcpcb *)(q->ptr); ! 263: register struct block *bp; ! 264: ! 265: while ((q->next->flag&QFULL) == 0 && (bp = getq(q)) != NULL) { ! 266: if(bp->type == M_DATA) ! 267: tp->so_rcount -= bp->wptr - bp->rptr; ! 268: if(tp->so_rcount < 0) ! 269: panic("so_rcount"); ! 270: (*q->next->qinfo->putp)(q->next, bp); ! 271: } ! 272: if(q->count <= q->qinfo->lolimit) ! 273: qenable(OTHERQ(q)); /* update remote send window */ ! 274: } ! 275: ! 276: /* ! 277: * here if channel isn't connected yet: ! 278: * output `data' is a user command ! 279: * grab all the data written so far, ! 280: * even though we'll discard the part we don't use ! 281: */ ! 282: tcpduser(tp) ! 283: register struct tcpcb *tp; ! 284: { ! 285: extern struct ipif *ip_ifwithaddr(); ! 286: struct tcpuser *tu; ! 287: struct block *bp, *tail, *head; ! 288: ! 289: tail = head = NULL; ! 290: while(bp = getq(tp->so_wq)){ ! 291: if(bp->type != M_DATA){ ! 292: freeb(bp); ! 293: continue; ! 294: } ! 295: bp->next = NULL; ! 296: if (head == NULL) ! 297: head = bp; ! 298: else ! 299: tail->next = bp; ! 300: tail = bp; ! 301: } ! 302: if(head == NULL) ! 303: return; ! 304: tp->so_wcount = 0; ! 305: bp = head; ! 306: if(BLEN(bp) < sizeof(struct tcpuser)){ ! 307: bp_free(bp); ! 308: return; ! 309: } ! 310: ! 311: /* ! 312: * tu->code is a request to bind this channel ! 313: */ ! 314: tu = (struct tcpuser *)bp->rptr; ! 315: switch(tu->code){ ! 316: case TCPC_CONNECT: ! 317: if(tp->t_state != TCPS_CLOSED){ ! 318: tu->code = TCPC_BADDEV; ! 319: goto bad; ! 320: } ! 321: if (tu->laddr != INADDR_ANY) { ! 322: /* has the user specified a legal local address? */ ! 323: if (ip_ifwithaddr(tu->laddr) == 0) { ! 324: tu->code = TCPC_BADLOCAL; ! 325: goto bad; ! 326: } ! 327: } else { ! 328: /* pick a local address related to the destination */ ! 329: tu->laddr = ip_hoston(tu->faddr); ! 330: if(tu->laddr == INADDR_ANY) { ! 331: tu->code = TCPC_NOROUTE; ! 332: goto bad; ! 333: } ! 334: } ! 335: tp->so_options = tu->param & ~SO_ACCEPTCONN; ! 336: if(tcpcb_bind(tp, tu->laddr, tu->lport)) { ! 337: tu->code = TCPC_BOUND; ! 338: goto bad; ! 339: } ! 340: tp->so_fport = tu->fport; ! 341: tp->so_faddr = tu->faddr; ! 342: tcp_template(tp); ! 343: if(tp->t_template == (struct block *)0) { ! 344: printf("no template - "); ! 345: tu->code = TCPC_BADDEV; ! 346: goto bad; ! 347: } ! 348: if (tp->so_options & SO_KEEPALIVE) ! 349: tcp_timers(tp, TCPT_KEEP); ! 350: tp->t_state = TCPS_SYN_SENT; ! 351: tp->iss = tcp_iss; tcp_iss += TCP_ISSINCR/2; ! 352: tcp_sendseqinit(tp); ! 353: tp->so_state &= ~SS_WAITING; ! 354: tcp_output(tp); ! 355: break; ! 356: case TCPC_LISTEN: ! 357: if(tp->t_state != TCPS_CLOSED){ ! 358: tu->code = TCPC_BADDEV; ! 359: goto bad; ! 360: } ! 361: if (tu->laddr != INADDR_ANY) { ! 362: /* has the user specified a legal local address? */ ! 363: if (ip_ifwithaddr(tu->laddr) == 0) { ! 364: tu->code = TCPC_BADLOCAL; ! 365: goto bad; ! 366: } ! 367: } ! 368: tp->so_options = tu->param & ~SO_ACCEPTCONN; ! 369: if(tcpcb_bind(tp, tu->laddr, tu->lport)) { ! 370: tu->code = TCPC_BOUND; ! 371: goto bad; ! 372: } ! 373: tp->t_state = TCPS_LISTEN; ! 374: tp->so_options |= SO_ACCEPTCONN; ! 375: tp->so_fport = tu->fport==0 ? TCPPORT_ANY : tu->fport; ! 376: tp->so_faddr = tu->faddr; ! 377: tcp_template(tp); ! 378: if(tp->t_template == 0) { ! 379: tu->code = TCPC_BADDEV; ! 380: printf("no template - "); ! 381: goto bad; ! 382: } ! 383: tp->so_state &= ~SS_WAITING; ! 384: if (tcp_isconnected(tp)<0) { ! 385: tu->code = TCPC_BADDEV; ! 386: goto bad; ! 387: } ! 388: break; ! 389: default: ! 390: tu->code = TCPC_SORRY; ! 391: goto bad; ! 392: } ! 393: bp_free(bp); ! 394: return; ! 395: bad: ! 396: /* ! 397: * send reason for failure back to user. ! 398: */ ! 399: if(bp->next) { ! 400: bp_free(bp->next); ! 401: bp->next = 0; ! 402: } ! 403: (*tp->so_rq->next->qinfo->putp)(tp->so_rq->next, bp); ! 404: tcp_hungup(tp); ! 405: } ! 406: ! 407: struct tcpcb * ! 408: tcp_disconnect(tp) ! 409: register struct tcpcb *tp; ! 410: { ! 411: ! 412: if(tp->t_state < TCPS_ESTABLISHED) ! 413: tp = tcp_close(tp); ! 414: else { ! 415: tp->so_state &= ~SS_PLEASEOPEN; ! 416: tcp_hungup(tp); /* sends M_HANGUP */ ! 417: tp = tcp_usrclosed(tp); ! 418: if(tp) ! 419: tcp_output(tp); ! 420: } ! 421: return(tp); ! 422: } ! 423: ! 424: struct tcpcb * ! 425: tcp_usrclosed(tp) ! 426: register struct tcpcb *tp; ! 427: { ! 428: ! 429: switch(tp->t_state){ ! 430: ! 431: case TCPS_CLOSED: ! 432: case TCPS_LISTEN: ! 433: case TCPS_SYN_SENT: ! 434: tp->t_state = TCPS_CLOSED; ! 435: tp = tcp_close(tp); ! 436: break; ! 437: ! 438: case TCPS_SYN_RECEIVED: ! 439: case TCPS_ESTABLISHED: ! 440: tp->t_state = TCPS_FIN_WAIT_1; ! 441: tp->so_options |= SO_KEEPALIVE; ! 442: tcp_timers(tp, TCPT_KEEP); ! 443: break; ! 444: ! 445: case TCPS_CLOSE_WAIT: ! 446: tp->t_state = TCPS_LAST_ACK; ! 447: tp->so_options |= SO_KEEPALIVE; ! 448: tcp_timers(tp, TCPT_KEEP); ! 449: break; ! 450: } ! 451: if(tp && tp->t_state >= TCPS_FIN_WAIT_2) { ! 452: tp->so_state &= ~(SS_PLEASEOPEN|SS_RCVATMARK); ! 453: if(tp->so_state&SS_OPEN && !(tp->so_state&SS_HUNGUP)) { ! 454: tp->so_state |= SS_HUNGUP; ! 455: tcp_hungup(tp); ! 456: } ! 457: } ! 458: return(tp); ! 459: } ! 460: ! 461: tcp_isconnected(tp) ! 462: struct tcpcb *tp; ! 463: { ! 464: struct block *bp; ! 465: struct tcpuser *tu; ! 466: struct tcpcb *rtp; ! 467: ! 468: if(tp->so_head) ! 469: rtp = tp->so_head; ! 470: else ! 471: rtp = tp; ! 472: if((rtp->so_state & SS_OPEN) == 0){ ! 473: printf("isconnected, no fd ref\n"); ! 474: return -1; ! 475: } ! 476: bp = allocb(sizeof(struct tcpuser)); ! 477: if(bp == 0) ! 478: return -1; ! 479: bp->wptr += sizeof(struct tcpuser); ! 480: tu = (struct tcpuser *)bp->rptr; ! 481: tu->code = TCPC_OK; ! 482: tu->fport = tp->so_fport; ! 483: tu->faddr = tp->so_faddr; ! 484: tu->lport = tp->so_lport; ! 485: tu->laddr = tp->so_laddr; ! 486: tu->param = tp->so_dev; ! 487: bp->class |= S_DELIM; ! 488: return tcpdrint(bp, rtp); ! 489: } ! 490: ! 491: tcp_hungup(tp) ! 492: register struct tcpcb *tp; ! 493: { ! 494: register struct queue *q; ! 495: ! 496: q = tp->so_rq; ! 497: if(q == 0) ! 498: return; ! 499: qpctl(q, M_HANGUP); ! 500: } ! 501: ! 502: /* ! 503: * find a spare even-numbered tcp device for a new passive-end ! 504: * connection. ! 505: */ ! 506: struct tcpcb * ! 507: tcp_newconn(tp) ! 508: register struct tcpcb *tp; ! 509: { ! 510: register struct tcpcb *ntp; ! 511: register struct tcpcb *tend; ! 512: ! 513: if(tp->so_rq && (tp->so_rq->flag&QFULL)){ ! 514: printf("listen %d q full\n", tp->so_lport); ! 515: return(0); ! 516: } ! 517: tend = &tcpcb[tcpcnt]; ! 518: for(ntp = &tcpcb[0]; ntp < tend; ntp += 2){ ! 519: if(ntp->t_state == TCPS_CLOSED){ ! 520: bzero(ntp, sizeof(struct tcpcb)); ! 521: ntp->t_maxseg = TCP_DEFMAXSEG; ! 522: ntp->so_options = tp->so_options & SO_KEEPALIVE; ! 523: ntp->so_head = tp; ! 524: ntp->so_dev = ntp - tcpcb; ! 525: return(ntp); ! 526: } ! 527: } ! 528: return(0); ! 529: } ! 530: ! 531: struct tcpcb * ! 532: tcpcb_lookup(faddr, fport, laddr, lport) ! 533: register in_addr faddr, laddr; ! 534: register tcp_port fport, lport; ! 535: { ! 536: register struct tcpcb *tp; ! 537: int highscore = 0, score; ! 538: struct tcpcb *match = 0; ! 539: struct tcpcb *tend; ! 540: ! 541: tend = &tcpcb[tcpcnt]; ! 542: for(tp = &tcpcb[0]; tp < tend; tp++){ ! 543: if(tp->t_state == TCPS_CLOSED) ! 544: continue; ! 545: score = 22; ! 546: if(tp->so_faddr != faddr) { ! 547: if(tp->so_faddr != INADDR_ANY) ! 548: continue; ! 549: else ! 550: score -=8; ! 551: } ! 552: if(tp->so_fport != fport) { ! 553: if(tp->so_fport != TCPPORT_ANY) ! 554: continue; ! 555: else ! 556: score -=8; ! 557: } ! 558: if(tp->so_laddr != laddr) { ! 559: if(tp->so_laddr == in_netof(laddr)) ! 560: score -=1; ! 561: else if(tp->so_laddr != INADDR_ANY) ! 562: continue; ! 563: else ! 564: score -=2; ! 565: } ! 566: if(tp->so_lport != lport) { ! 567: if(tp->so_lport != TCPPORT_ANY) ! 568: continue; ! 569: else ! 570: score -=4; ! 571: } ! 572: if (score==22) ! 573: return tp; ! 574: if (score<highscore) ! 575: continue; ! 576: match = tp; ! 577: highscore = score; ! 578: } ! 579: return(match); ! 580: } ! 581: ! 582: /* n chars were acked; drop them now */ ! 583: sbsnddrop(tp, n) ! 584: register struct tcpcb *tp; ! 585: register int n; ! 586: { ! 587: register struct queue *q; ! 588: register int i; ! 589: register struct block *bp; ! 590: ! 591: q = tp->so_wq; ! 592: if(q == 0) ! 593: return; ! 594: bp = 0; ! 595: while(n > 0 && (bp = getq(q))){ ! 596: i = MIN(BLEN(bp), n); ! 597: bp->rptr += i; ! 598: n -= i; ! 599: tp->so_wcount -= i; ! 600: if(bp->rptr >= bp->wptr){ ! 601: freeb(bp); ! 602: bp = 0; ! 603: } else if(n > 0){ ! 604: panic("sbsnddrop"); ! 605: } ! 606: } ! 607: if(bp) ! 608: putbq(q, bp); ! 609: } ! 610: ! 611: static tcp_port portnext[] = { 600, 1024 }; ! 612: static tcp_port portlow[] = { 600, 1024 }; ! 613: static tcp_port porthigh[] = { 1024, 2048 }; ! 614: ! 615: tcpcb_bind(tp, addr, port) ! 616: register struct tcpcb *tp; ! 617: register in_addr addr; ! 618: register tcp_port port; ! 619: { ! 620: register int i; ! 621: ! 622: tp->so_lport = 0; ! 623: if(port){ ! 624: /* ! 625: * Don't let just anyone get a trusted port. ! 626: */ ! 627: if(port<porthigh[0] && u.u_uid!=0) ! 628: return(1); ! 629: tp->so_lport = port; ! 630: tp->so_laddr = addr; ! 631: return(0); ! 632: } ! 633: /* ! 634: * No specific port, pick one. Root only gets a trusted ! 635: * port if explicitly requested, so privileged programs ! 636: * can make non-privileged calls. ! 637: */ ! 638: i = (u.u_uid==0 && (tp->so_options&SO_TRUSTED)) ? 0 : 1; ! 639: if(portnext[i] >= porthigh[i]) ! 640: portnext[i] = portlow[i]; ! 641: port = portnext[i]; ! 642: while(1){ ! 643: if(tcpcb_bind(tp, addr, portnext[i]) == 0){ ! 644: portnext[i]++; ! 645: return(0); ! 646: } ! 647: portnext[i]++; ! 648: if(portnext[i] >= porthigh[i]) ! 649: portnext[i] = portlow[i]; ! 650: if(portnext[i] == port) /* tried them all */ ! 651: break; ! 652: } ! 653: return(1); ! 654: } ! 655: ! 656: tcp_cantrcvmore(tp) ! 657: register struct tcpcb *tp; ! 658: { ! 659: register struct queue *q; ! 660: ! 661: q = tp->so_rq; ! 662: if(q == NULL) ! 663: return; ! 664: if(tp->so_state & SS_HANGUP) ! 665: qpctl(q, M_HANGUP); ! 666: else { ! 667: /* two delims ensure a zero length read at the process */ ! 668: qpctld(q, M_DATA); ! 669: qpctld(q, M_DATA); ! 670: } ! 671: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.