![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to socket.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Qemu by Fabrice Bellard] / qemu / slirp|
Return to socket.c CVS log | Up to [Qemu by Fabrice Bellard] / qemu / slirp |
1.1 root 1: /*
2: * Copyright (c) 1995 Danny Gasparovski.
3: *
4: * Please read the file COPYRIGHT for the
5: * terms and conditions of the copyright.
6: */
7:
8: #define WANT_SYS_IOCTL_H
9: #include <slirp.h>
10: #include "ip_icmp.h"
11: #include "main.h"
1.1.1.2 ! root 12: #ifdef __sun__
! 13: #include <sys/filio.h>
! 14: #endif
1.1 root 15:
16: void
17: so_init()
18: {
19: /* Nothing yet */
20: }
21:
22:
23: struct socket *
24: solookup(head, laddr, lport, faddr, fport)
25: struct socket *head;
26: struct in_addr laddr;
27: u_int lport;
28: struct in_addr faddr;
29: u_int fport;
30: {
31: struct socket *so;
32:
33: for (so = head->so_next; so != head; so = so->so_next) {
34: if (so->so_lport == lport &&
35: so->so_laddr.s_addr == laddr.s_addr &&
36: so->so_faddr.s_addr == faddr.s_addr &&
37: so->so_fport == fport)
38: break;
39: }
40:
41: if (so == head)
42: return (struct socket *)NULL;
43: return so;
44:
45: }
46:
47: /*
48: * Create a new socket, initialise the fields
49: * It is the responsibility of the caller to
50: * insque() it into the correct linked-list
51: */
52: struct socket *
53: socreate()
54: {
55: struct socket *so;
56:
57: so = (struct socket *)malloc(sizeof(struct socket));
58: if(so) {
59: memset(so, 0, sizeof(struct socket));
60: so->so_state = SS_NOFDREF;
61: so->s = -1;
62: }
63: return(so);
64: }
65:
66: /*
67: * remque and free a socket, clobber cache
68: */
69: void
70: sofree(so)
71: struct socket *so;
72: {
73: if (so->so_emu==EMU_RSH && so->extra) {
74: sofree(so->extra);
75: so->extra=NULL;
76: }
77: if (so == tcp_last_so)
78: tcp_last_so = &tcb;
79: else if (so == udp_last_so)
80: udp_last_so = &udb;
81:
82: m_free(so->so_m);
83:
84: if(so->so_next && so->so_prev)
85: remque(so); /* crashes if so is not in a queue */
86:
87: free(so);
88: }
89:
90: /*
91: * Read from so's socket into sb_snd, updating all relevant sbuf fields
92: * NOTE: This will only be called if it is select()ed for reading, so
93: * a read() of 0 (or less) means it's disconnected
94: */
95: int
96: soread(so)
97: struct socket *so;
98: {
99: int n, nn, lss, total;
100: struct sbuf *sb = &so->so_snd;
101: int len = sb->sb_datalen - sb->sb_cc;
102: struct iovec iov[2];
103: int mss = so->so_tcpcb->t_maxseg;
104:
105: DEBUG_CALL("soread");
106: DEBUG_ARG("so = %lx", (long )so);
107:
108: /*
109: * No need to check if there's enough room to read.
110: * soread wouldn't have been called if there weren't
111: */
112:
113: len = sb->sb_datalen - sb->sb_cc;
114:
115: iov[0].iov_base = sb->sb_wptr;
116: if (sb->sb_wptr < sb->sb_rptr) {
117: iov[0].iov_len = sb->sb_rptr - sb->sb_wptr;
118: /* Should never succeed, but... */
119: if (iov[0].iov_len > len)
120: iov[0].iov_len = len;
121: if (iov[0].iov_len > mss)
122: iov[0].iov_len -= iov[0].iov_len%mss;
123: n = 1;
124: } else {
125: iov[0].iov_len = (sb->sb_data + sb->sb_datalen) - sb->sb_wptr;
126: /* Should never succeed, but... */
127: if (iov[0].iov_len > len) iov[0].iov_len = len;
128: len -= iov[0].iov_len;
129: if (len) {
130: iov[1].iov_base = sb->sb_data;
131: iov[1].iov_len = sb->sb_rptr - sb->sb_data;
132: if(iov[1].iov_len > len)
133: iov[1].iov_len = len;
134: total = iov[0].iov_len + iov[1].iov_len;
135: if (total > mss) {
136: lss = total%mss;
137: if (iov[1].iov_len > lss) {
138: iov[1].iov_len -= lss;
139: n = 2;
140: } else {
141: lss -= iov[1].iov_len;
142: iov[0].iov_len -= lss;
143: n = 1;
144: }
145: } else
146: n = 2;
147: } else {
148: if (iov[0].iov_len > mss)
149: iov[0].iov_len -= iov[0].iov_len%mss;
150: n = 1;
151: }
152: }
153:
154: #ifdef HAVE_READV
155: nn = readv(so->s, (struct iovec *)iov, n);
156: DEBUG_MISC((dfd, " ... read nn = %d bytes\n", nn));
157: #else
158: nn = recv(so->s, iov[0].iov_base, iov[0].iov_len,0);
159: #endif
160: if (nn <= 0) {
161: if (nn < 0 && (errno == EINTR || errno == EAGAIN))
162: return 0;
163: else {
164: DEBUG_MISC((dfd, " --- soread() disconnected, nn = %d, errno = %d-%s\n", nn, errno,strerror(errno)));
165: sofcantrcvmore(so);
166: tcp_sockclosed(sototcpcb(so));
167: return -1;
168: }
169: }
170:
171: #ifndef HAVE_READV
172: /*
173: * If there was no error, try and read the second time round
174: * We read again if n = 2 (ie, there's another part of the buffer)
175: * and we read as much as we could in the first read
176: * We don't test for <= 0 this time, because there legitimately
177: * might not be any more data (since the socket is non-blocking),
178: * a close will be detected on next iteration.
179: * A return of -1 wont (shouldn't) happen, since it didn't happen above
180: */
181: if (n == 2 && nn == iov[0].iov_len) {
182: int ret;
183: ret = recv(so->s, iov[1].iov_base, iov[1].iov_len,0);
184: if (ret > 0)
185: nn += ret;
186: }
187:
188: DEBUG_MISC((dfd, " ... read nn = %d bytes\n", nn));
189: #endif
190:
191: /* Update fields */
192: sb->sb_cc += nn;
193: sb->sb_wptr += nn;
194: if (sb->sb_wptr >= (sb->sb_data + sb->sb_datalen))
195: sb->sb_wptr -= sb->sb_datalen;
196: return nn;
197: }
198:
199: /*
200: * Get urgent data
201: *
202: * When the socket is created, we set it SO_OOBINLINE,
203: * so when OOB data arrives, we soread() it and everything
204: * in the send buffer is sent as urgent data
205: */
206: void
207: sorecvoob(so)
208: struct socket *so;
209: {
210: struct tcpcb *tp = sototcpcb(so);
211:
212: DEBUG_CALL("sorecvoob");
213: DEBUG_ARG("so = %lx", (long)so);
214:
215: /*
216: * We take a guess at how much urgent data has arrived.
217: * In most situations, when urgent data arrives, the next
218: * read() should get all the urgent data. This guess will
219: * be wrong however if more data arrives just after the
220: * urgent data, or the read() doesn't return all the
221: * urgent data.
222: */
223: soread(so);
224: tp->snd_up = tp->snd_una + so->so_snd.sb_cc;
225: tp->t_force = 1;
226: tcp_output(tp);
227: tp->t_force = 0;
228: }
229:
230: /*
231: * Send urgent data
232: * There's a lot duplicated code here, but...
233: */
234: int
235: sosendoob(so)
236: struct socket *so;
237: {
238: struct sbuf *sb = &so->so_rcv;
239: char buff[2048]; /* XXX Shouldn't be sending more oob data than this */
240:
241: int n, len;
242:
243: DEBUG_CALL("sosendoob");
244: DEBUG_ARG("so = %lx", (long)so);
245: DEBUG_ARG("sb->sb_cc = %d", sb->sb_cc);
246:
247: if (so->so_urgc > 2048)
248: so->so_urgc = 2048; /* XXXX */
249:
250: if (sb->sb_rptr < sb->sb_wptr) {
251: /* We can send it directly */
252: n = send(so->s, sb->sb_rptr, so->so_urgc, (MSG_OOB)); /* |MSG_DONTWAIT)); */
253: so->so_urgc -= n;
254:
255: DEBUG_MISC((dfd, " --- sent %d bytes urgent data, %d urgent bytes left\n", n, so->so_urgc));
256: } else {
257: /*
258: * Since there's no sendv or sendtov like writev,
259: * we must copy all data to a linear buffer then
260: * send it all
261: */
262: len = (sb->sb_data + sb->sb_datalen) - sb->sb_rptr;
263: if (len > so->so_urgc) len = so->so_urgc;
264: memcpy(buff, sb->sb_rptr, len);
265: so->so_urgc -= len;
266: if (so->so_urgc) {
267: n = sb->sb_wptr - sb->sb_data;
268: if (n > so->so_urgc) n = so->so_urgc;
269: memcpy((buff + len), sb->sb_data, n);
270: so->so_urgc -= n;
271: len += n;
272: }
273: n = send(so->s, buff, len, (MSG_OOB)); /* |MSG_DONTWAIT)); */
274: #ifdef DEBUG
275: if (n != len)
276: DEBUG_ERROR((dfd, "Didn't send all data urgently XXXXX\n"));
277: #endif
278: DEBUG_MISC((dfd, " ---2 sent %d bytes urgent data, %d urgent bytes left\n", n, so->so_urgc));
279: }
280:
281: sb->sb_cc -= n;
282: sb->sb_rptr += n;
283: if (sb->sb_rptr >= (sb->sb_data + sb->sb_datalen))
284: sb->sb_rptr -= sb->sb_datalen;
285:
286: return n;
287: }
288:
289: /*
290: * Write data from so_rcv to so's socket,
291: * updating all sbuf field as necessary
292: */
293: int
294: sowrite(so)
295: struct socket *so;
296: {
297: int n,nn;
298: struct sbuf *sb = &so->so_rcv;
299: int len = sb->sb_cc;
300: struct iovec iov[2];
301:
302: DEBUG_CALL("sowrite");
303: DEBUG_ARG("so = %lx", (long)so);
304:
305: if (so->so_urgc) {
306: sosendoob(so);
307: if (sb->sb_cc == 0)
308: return 0;
309: }
310:
311: /*
312: * No need to check if there's something to write,
313: * sowrite wouldn't have been called otherwise
314: */
315:
316: len = sb->sb_cc;
317:
318: iov[0].iov_base = sb->sb_rptr;
319: if (sb->sb_rptr < sb->sb_wptr) {
320: iov[0].iov_len = sb->sb_wptr - sb->sb_rptr;
321: /* Should never succeed, but... */
322: if (iov[0].iov_len > len) iov[0].iov_len = len;
323: n = 1;
324: } else {
325: iov[0].iov_len = (sb->sb_data + sb->sb_datalen) - sb->sb_rptr;
326: if (iov[0].iov_len > len) iov[0].iov_len = len;
327: len -= iov[0].iov_len;
328: if (len) {
329: iov[1].iov_base = sb->sb_data;
330: iov[1].iov_len = sb->sb_wptr - sb->sb_data;
331: if (iov[1].iov_len > len) iov[1].iov_len = len;
332: n = 2;
333: } else
334: n = 1;
335: }
336: /* Check if there's urgent data to send, and if so, send it */
337:
338: #ifdef HAVE_READV
339: nn = writev(so->s, (const struct iovec *)iov, n);
340:
341: DEBUG_MISC((dfd, " ... wrote nn = %d bytes\n", nn));
342: #else
343: nn = send(so->s, iov[0].iov_base, iov[0].iov_len,0);
344: #endif
345: /* This should never happen, but people tell me it does *shrug* */
346: if (nn < 0 && (errno == EAGAIN || errno == EINTR))
347: return 0;
348:
349: if (nn <= 0) {
350: DEBUG_MISC((dfd, " --- sowrite disconnected, so->so_state = %x, errno = %d\n",
351: so->so_state, errno));
352: sofcantsendmore(so);
353: tcp_sockclosed(sototcpcb(so));
354: return -1;
355: }
356:
357: #ifndef HAVE_READV
358: if (n == 2 && nn == iov[0].iov_len) {
359: int ret;
360: ret = send(so->s, iov[1].iov_base, iov[1].iov_len,0);
361: if (ret > 0)
362: nn += ret;
363: }
364: DEBUG_MISC((dfd, " ... wrote nn = %d bytes\n", nn));
365: #endif
366:
367: /* Update sbuf */
368: sb->sb_cc -= nn;
369: sb->sb_rptr += nn;
370: if (sb->sb_rptr >= (sb->sb_data + sb->sb_datalen))
371: sb->sb_rptr -= sb->sb_datalen;
372:
373: /*
374: * If in DRAIN mode, and there's no more data, set
375: * it CANTSENDMORE
376: */
377: if ((so->so_state & SS_FWDRAIN) && sb->sb_cc == 0)
378: sofcantsendmore(so);
379:
380: return nn;
381: }
382:
383: /*
384: * recvfrom() a UDP socket
385: */
386: void
387: sorecvfrom(so)
388: struct socket *so;
389: {
390: struct sockaddr_in addr;
391: int addrlen = sizeof(struct sockaddr_in);
392:
393: DEBUG_CALL("sorecvfrom");
394: DEBUG_ARG("so = %lx", (long)so);
395:
396: if (so->so_type == IPPROTO_ICMP) { /* This is a "ping" reply */
397: char buff[256];
398: int len;
399:
400: len = recvfrom(so->s, buff, 256, 0,
401: (struct sockaddr *)&addr, &addrlen);
402: /* XXX Check if reply is "correct"? */
403:
404: if(len == -1 || len == 0) {
405: u_char code=ICMP_UNREACH_PORT;
406:
407: if(errno == EHOSTUNREACH) code=ICMP_UNREACH_HOST;
408: else if(errno == ENETUNREACH) code=ICMP_UNREACH_NET;
409:
410: DEBUG_MISC((dfd," udp icmp rx errno = %d-%s\n",
411: errno,strerror(errno)));
412: icmp_error(so->so_m, ICMP_UNREACH,code, 0,strerror(errno));
413: } else {
414: icmp_reflect(so->so_m);
415: so->so_m = 0; /* Don't m_free() it again! */
416: }
417: /* No need for this socket anymore, udp_detach it */
418: udp_detach(so);
419: } else { /* A "normal" UDP packet */
420: struct mbuf *m;
421: int len, n;
422:
423: if (!(m = m_get())) return;
424: m->m_data += if_maxlinkhdr;
425:
426: /*
427: * XXX Shouldn't FIONREAD packets destined for port 53,
428: * but I don't know the max packet size for DNS lookups
429: */
430: len = M_FREEROOM(m);
431: /* if (so->so_fport != htons(53)) { */
432: ioctlsocket(so->s, FIONREAD, &n);
433:
434: if (n > len) {
435: n = (m->m_data - m->m_dat) + m->m_len + n + 1;
436: m_inc(m, n);
437: len = M_FREEROOM(m);
438: }
439: /* } */
440:
441: m->m_len = recvfrom(so->s, m->m_data, len, 0,
442: (struct sockaddr *)&addr, &addrlen);
443: DEBUG_MISC((dfd, " did recvfrom %d, errno = %d-%s\n",
444: m->m_len, errno,strerror(errno)));
445: if(m->m_len<0) {
446: u_char code=ICMP_UNREACH_PORT;
447:
448: if(errno == EHOSTUNREACH) code=ICMP_UNREACH_HOST;
449: else if(errno == ENETUNREACH) code=ICMP_UNREACH_NET;
450:
451: DEBUG_MISC((dfd," rx error, tx icmp ICMP_UNREACH:%i\n", code));
452: icmp_error(so->so_m, ICMP_UNREACH,code, 0,strerror(errno));
453: m_free(m);
454: } else {
455: /*
456: * Hack: domain name lookup will be used the most for UDP,
457: * and since they'll only be used once there's no need
458: * for the 4 minute (or whatever) timeout... So we time them
459: * out much quicker (10 seconds for now...)
460: */
461: if (so->so_expire) {
462: if (so->so_fport == htons(53))
463: so->so_expire = curtime + SO_EXPIREFAST;
464: else
465: so->so_expire = curtime + SO_EXPIRE;
466: }
467:
468: /* if (m->m_len == len) {
469: * m_inc(m, MINCSIZE);
470: * m->m_len = 0;
471: * }
472: */
473:
474: /*
475: * If this packet was destined for CTL_ADDR,
476: * make it look like that's where it came from, done by udp_output
477: */
478: udp_output(so, m, &addr);
479: } /* rx error */
480: } /* if ping packet */
481: }
482:
483: /*
484: * sendto() a socket
485: */
486: int
487: sosendto(so, m)
488: struct socket *so;
489: struct mbuf *m;
490: {
491: int ret;
492: struct sockaddr_in addr;
493:
494: DEBUG_CALL("sosendto");
495: DEBUG_ARG("so = %lx", (long)so);
496: DEBUG_ARG("m = %lx", (long)m);
497:
498: addr.sin_family = AF_INET;
499: if ((so->so_faddr.s_addr & htonl(0xffffff00)) == special_addr.s_addr) {
500: /* It's an alias */
501: switch(ntohl(so->so_faddr.s_addr) & 0xff) {
502: case CTL_DNS:
503: addr.sin_addr = dns_addr;
504: break;
505: case CTL_ALIAS:
506: default:
507: addr.sin_addr = loopback_addr;
508: break;
509: }
510: } else
511: addr.sin_addr = so->so_faddr;
512: addr.sin_port = so->so_fport;
513:
514: DEBUG_MISC((dfd, " sendto()ing, addr.sin_port=%d, addr.sin_addr.s_addr=%.16s\n", ntohs(addr.sin_port), inet_ntoa(addr.sin_addr)));
515:
516: /* Don't care what port we get */
517: ret = sendto(so->s, m->m_data, m->m_len, 0,
518: (struct sockaddr *)&addr, sizeof (struct sockaddr));
519: if (ret < 0)
520: return -1;
521:
522: /*
523: * Kill the socket if there's no reply in 4 minutes,
524: * but only if it's an expirable socket
525: */
526: if (so->so_expire)
527: so->so_expire = curtime + SO_EXPIRE;
528: so->so_state = SS_ISFCONNECTED; /* So that it gets select()ed */
529: return 0;
530: }
531:
532: /*
533: * XXX This should really be tcp_listen
534: */
535: struct socket *
536: solisten(port, laddr, lport, flags)
537: u_int port;
538: u_int32_t laddr;
539: u_int lport;
540: int flags;
541: {
542: struct sockaddr_in addr;
543: struct socket *so;
544: int s, addrlen = sizeof(addr), opt = 1;
545:
546: DEBUG_CALL("solisten");
547: DEBUG_ARG("port = %d", port);
548: DEBUG_ARG("laddr = %x", laddr);
549: DEBUG_ARG("lport = %d", lport);
550: DEBUG_ARG("flags = %x", flags);
551:
552: if ((so = socreate()) == NULL) {
553: /* free(so); Not sofree() ??? free(NULL) == NOP */
554: return NULL;
555: }
556:
557: /* Don't tcp_attach... we don't need so_snd nor so_rcv */
558: if ((so->so_tcpcb = tcp_newtcpcb(so)) == NULL) {
559: free(so);
560: return NULL;
561: }
562: insque(so,&tcb);
563:
564: /*
565: * SS_FACCEPTONCE sockets must time out.
566: */
567: if (flags & SS_FACCEPTONCE)
568: so->so_tcpcb->t_timer[TCPT_KEEP] = TCPTV_KEEP_INIT*2;
569:
570: so->so_state = (SS_FACCEPTCONN|flags);
571: so->so_lport = lport; /* Kept in network format */
572: so->so_laddr.s_addr = laddr; /* Ditto */
573:
574: addr.sin_family = AF_INET;
575: addr.sin_addr.s_addr = INADDR_ANY;
576: addr.sin_port = port;
577:
578: if (((s = socket(AF_INET,SOCK_STREAM,0)) < 0) ||
1.1.1.2 ! root 579: (setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(char *)&opt,sizeof(int)) < 0) ||
1.1 root 580: (bind(s,(struct sockaddr *)&addr, sizeof(addr)) < 0) ||
581: (listen(s,1) < 0)) {
582: int tmperrno = errno; /* Don't clobber the real reason we failed */
583:
584: close(s);
585: sofree(so);
586: /* Restore the real errno */
587: #ifdef _WIN32
588: WSASetLastError(tmperrno);
589: #else
590: errno = tmperrno;
591: #endif
592: return NULL;
593: }
594: setsockopt(s,SOL_SOCKET,SO_OOBINLINE,(char *)&opt,sizeof(int));
595:
596: getsockname(s,(struct sockaddr *)&addr,&addrlen);
597: so->so_fport = addr.sin_port;
598: if (addr.sin_addr.s_addr == 0 || addr.sin_addr.s_addr == loopback_addr.s_addr)
1.1.1.2 ! root 599: so->so_faddr = alias_addr;
1.1 root 600: else
601: so->so_faddr = addr.sin_addr;
602:
603: so->s = s;
604: return so;
605: }
606:
607: /*
608: * Data is available in so_rcv
609: * Just write() the data to the socket
610: * XXX not yet...
611: */
612: void
613: sorwakeup(so)
614: struct socket *so;
615: {
616: /* sowrite(so); */
617: /* FD_CLR(so->s,&writefds); */
618: }
619:
620: /*
621: * Data has been freed in so_snd
622: * We have room for a read() if we want to
623: * For now, don't read, it'll be done in the main loop
624: */
625: void
626: sowwakeup(so)
627: struct socket *so;
628: {
629: /* Nothing, yet */
630: }
631:
632: /*
633: * Various session state calls
634: * XXX Should be #define's
635: * The socket state stuff needs work, these often get call 2 or 3
636: * times each when only 1 was needed
637: */
638: void
639: soisfconnecting(so)
640: register struct socket *so;
641: {
642: so->so_state &= ~(SS_NOFDREF|SS_ISFCONNECTED|SS_FCANTRCVMORE|
643: SS_FCANTSENDMORE|SS_FWDRAIN);
644: so->so_state |= SS_ISFCONNECTING; /* Clobber other states */
645: }
646:
647: void
648: soisfconnected(so)
649: register struct socket *so;
650: {
651: so->so_state &= ~(SS_ISFCONNECTING|SS_FWDRAIN|SS_NOFDREF);
652: so->so_state |= SS_ISFCONNECTED; /* Clobber other states */
653: }
654:
655: void
656: sofcantrcvmore(so)
657: struct socket *so;
658: {
659: if ((so->so_state & SS_NOFDREF) == 0) {
660: shutdown(so->s,0);
661: if(global_writefds) {
662: FD_CLR(so->s,global_writefds);
663: }
664: }
665: so->so_state &= ~(SS_ISFCONNECTING);
666: if (so->so_state & SS_FCANTSENDMORE)
667: so->so_state = SS_NOFDREF; /* Don't select it */ /* XXX close() here as well? */
668: else
669: so->so_state |= SS_FCANTRCVMORE;
670: }
671:
672: void
673: sofcantsendmore(so)
674: struct socket *so;
675: {
676: if ((so->so_state & SS_NOFDREF) == 0) {
677: shutdown(so->s,1); /* send FIN to fhost */
678: if (global_readfds) {
679: FD_CLR(so->s,global_readfds);
680: }
681: if (global_xfds) {
682: FD_CLR(so->s,global_xfds);
683: }
684: }
685: so->so_state &= ~(SS_ISFCONNECTING);
686: if (so->so_state & SS_FCANTRCVMORE)
687: so->so_state = SS_NOFDREF; /* as above */
688: else
689: so->so_state |= SS_FCANTSENDMORE;
690: }
691:
692: void
693: soisfdisconnected(so)
694: struct socket *so;
695: {
696: /* so->so_state &= ~(SS_ISFCONNECTING|SS_ISFCONNECTED); */
697: /* close(so->s); */
698: /* so->so_state = SS_ISFDISCONNECTED; */
699: /*
700: * XXX Do nothing ... ?
701: */
702: }
703:
704: /*
705: * Set write drain mode
706: * Set CANTSENDMORE once all data has been write()n
707: */
708: void
709: sofwdrain(so)
710: struct socket *so;
711: {
712: if (so->so_rcv.sb_cc)
713: so->so_state |= SS_FWDRAIN;
714: else
715: sofcantsendmore(so);
716: }
717:
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.