|
|
1.1 root 1: /*
2: * Copyright (c) 1982, 1986, 1988, 1990, 1993
3: * The Regents of the University of California. All rights reserved.
4: *
5: * Redistribution and use in source and binary forms, with or without
6: * modification, are permitted provided that the following conditions
7: * are met:
8: * 1. Redistributions of source code must retain the above copyright
9: * notice, this list of conditions and the following disclaimer.
10: * 2. Redistributions in binary form must reproduce the above copyright
11: * notice, this list of conditions and the following disclaimer in the
12: * documentation and/or other materials provided with the distribution.
13: * 3. All advertising materials mentioning features or use of this software
14: * must display the following acknowledgement:
15: * This product includes software developed by the University of
16: * California, Berkeley and its contributors.
17: * 4. Neither the name of the University nor the names of its contributors
18: * may be used to endorse or promote products derived from this software
19: * without specific prior written permission.
20: *
21: * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
22: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24: * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
25: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
26: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
27: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
28: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
29: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
30: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
31: * SUCH DAMAGE.
32: *
33: * @(#)udp_usrreq.c 8.4 (Berkeley) 1/21/94
34: * udp_usrreq.c,v 1.4 1994/10/02 17:48:45 phk Exp
35: */
36:
37: /*
38: * Changes and additions relating to SLiRP
39: * Copyright (c) 1995 Danny Gasparovski.
40: *
41: * Please read the file COPYRIGHT for the
42: * terms and conditions of the copyright.
43: */
44:
45: #include <slirp.h>
46: #include "ip_icmp.h"
47:
48: struct udpstat udpstat;
49:
50: struct socket udb;
51:
52: /*
53: * UDP protocol implementation.
54: * Per RFC 768, August, 1980.
55: */
56: #ifndef COMPAT_42
57: int udpcksum = 1;
58: #else
59: int udpcksum = 0; /* XXX */
60: #endif
61:
62: struct socket *udp_last_so = &udb;
63:
64: void
65: udp_init()
66: {
67: udb.so_next = udb.so_prev = &udb;
68: }
69: /* m->m_data points at ip packet header
70: * m->m_len length ip packet
71: * ip->ip_len length data (IPDU)
72: */
73: void
74: udp_input(m, iphlen)
75: register struct mbuf *m;
76: int iphlen;
77: {
78: register struct ip *ip;
79: register struct udphdr *uh;
80: /* struct mbuf *opts = 0;*/
81: int len;
82: struct ip save_ip;
83: struct socket *so;
84:
85: DEBUG_CALL("udp_input");
86: DEBUG_ARG("m = %lx", (long)m);
87: DEBUG_ARG("iphlen = %d", iphlen);
88:
89: udpstat.udps_ipackets++;
90:
91: /*
92: * Strip IP options, if any; should skip this,
93: * make available to user, and use on returned packets,
94: * but we don't yet have a way to check the checksum
95: * with options still present.
96: */
97: if(iphlen > sizeof(struct ip)) {
98: ip_stripoptions(m, (struct mbuf *)0);
99: iphlen = sizeof(struct ip);
100: }
101:
102: /*
103: * Get IP and UDP header together in first mbuf.
104: */
105: ip = mtod(m, struct ip *);
106: uh = (struct udphdr *)((caddr_t)ip + iphlen);
107:
108: /*
109: * Make mbuf data length reflect UDP length.
110: * If not enough data to reflect UDP length, drop.
111: */
112: len = ntohs((u_int16_t)uh->uh_ulen);
113:
114: if (ip->ip_len != len) {
115: if (len > ip->ip_len) {
116: udpstat.udps_badlen++;
117: goto bad;
118: }
119: m_adj(m, len - ip->ip_len);
120: ip->ip_len = len;
121: }
122:
123: /*
124: * Save a copy of the IP header in case we want restore it
125: * for sending an ICMP error message in response.
126: */
127: save_ip = *ip;
128: save_ip.ip_len+= iphlen; /* tcp_input subtracts this */
129:
130: /*
131: * Checksum extended UDP header and data.
132: */
133: if (udpcksum && uh->uh_sum) {
134: ((struct ipovly *)ip)->ih_next = 0;
135: ((struct ipovly *)ip)->ih_prev = 0;
136: ((struct ipovly *)ip)->ih_x1 = 0;
137: ((struct ipovly *)ip)->ih_len = uh->uh_ulen;
138: /* keep uh_sum for ICMP reply
139: * uh->uh_sum = cksum(m, len + sizeof (struct ip));
140: * if (uh->uh_sum) {
141: */
142: if(cksum(m, len + sizeof(struct ip))) {
143: udpstat.udps_badsum++;
144: goto bad;
145: }
146: }
147:
148: /*
149: * handle DHCP/BOOTP
150: */
151: if (ntohs(uh->uh_dport) == BOOTP_SERVER) {
152: bootp_input(m);
153: goto bad;
154: }
155:
156: /*
157: * handle TFTP
158: */
159: if (ntohs(uh->uh_dport) == TFTP_SERVER) {
160: tftp_input(m);
161: goto bad;
162: }
163:
164: /*
165: * Locate pcb for datagram.
166: */
167: so = udp_last_so;
168: if (so->so_lport != uh->uh_sport ||
169: so->so_laddr.s_addr != ip->ip_src.s_addr) {
170: struct socket *tmp;
171:
172: for (tmp = udb.so_next; tmp != &udb; tmp = tmp->so_next) {
173: if (tmp->so_lport == uh->uh_sport &&
174: tmp->so_laddr.s_addr == ip->ip_src.s_addr) {
175: tmp->so_faddr.s_addr = ip->ip_dst.s_addr;
176: tmp->so_fport = uh->uh_dport;
177: so = tmp;
178: break;
179: }
180: }
181: if (tmp == &udb) {
182: so = NULL;
183: } else {
184: udpstat.udpps_pcbcachemiss++;
185: udp_last_so = so;
186: }
187: }
188:
189: if (so == NULL) {
190: /*
191: * If there's no socket for this packet,
192: * create one
193: */
194: if ((so = socreate()) == NULL) goto bad;
195: if(udp_attach(so) == -1) {
196: DEBUG_MISC((dfd," udp_attach errno = %d-%s\n",
197: errno,strerror(errno)));
198: sofree(so);
199: goto bad;
200: }
201:
202: /*
203: * Setup fields
204: */
205: /* udp_last_so = so; */
206: so->so_laddr = ip->ip_src;
207: so->so_lport = uh->uh_sport;
208:
209: if ((so->so_iptos = udp_tos(so)) == 0)
210: so->so_iptos = ip->ip_tos;
211:
212: /*
213: * XXXXX Here, check if it's in udpexec_list,
214: * and if it is, do the fork_exec() etc.
215: */
216: }
217:
1.1.1.4 ! root 218: so->so_faddr = ip->ip_dst; /* XXX */
! 219: so->so_fport = uh->uh_dport; /* XXX */
! 220:
1.1 root 221: iphlen += sizeof(struct udphdr);
222: m->m_len -= iphlen;
223: m->m_data += iphlen;
224:
225: /*
226: * Now we sendto() the packet.
227: */
228: if (so->so_emu)
229: udp_emu(so, m);
230:
231: if(sosendto(so,m) == -1) {
232: m->m_len += iphlen;
233: m->m_data -= iphlen;
234: *ip=save_ip;
235: DEBUG_MISC((dfd,"udp tx errno = %d-%s\n",errno,strerror(errno)));
236: icmp_error(m, ICMP_UNREACH,ICMP_UNREACH_NET, 0,strerror(errno));
237: }
238:
239: m_free(so->so_m); /* used for ICMP if error on sorecvfrom */
240:
241: /* restore the orig mbuf packet */
242: m->m_len += iphlen;
243: m->m_data -= iphlen;
244: *ip=save_ip;
245: so->so_m=m; /* ICMP backup */
246:
247: return;
248: bad:
249: m_freem(m);
250: /* if (opts) m_freem(opts); */
251: return;
252: }
253:
254: int udp_output2(struct socket *so, struct mbuf *m,
255: struct sockaddr_in *saddr, struct sockaddr_in *daddr,
256: int iptos)
257: {
258: register struct udpiphdr *ui;
259: int error = 0;
260:
261: DEBUG_CALL("udp_output");
262: DEBUG_ARG("so = %lx", (long)so);
263: DEBUG_ARG("m = %lx", (long)m);
264: DEBUG_ARG("saddr = %lx", (long)saddr->sin_addr.s_addr);
265: DEBUG_ARG("daddr = %lx", (long)daddr->sin_addr.s_addr);
266:
267: /*
268: * Adjust for header
269: */
270: m->m_data -= sizeof(struct udpiphdr);
271: m->m_len += sizeof(struct udpiphdr);
272:
273: /*
274: * Fill in mbuf with extended UDP header
275: * and addresses and length put into network format.
276: */
277: ui = mtod(m, struct udpiphdr *);
278: ui->ui_next = ui->ui_prev = 0;
279: ui->ui_x1 = 0;
280: ui->ui_pr = IPPROTO_UDP;
281: ui->ui_len = htons(m->m_len - sizeof(struct ip)); /* + sizeof (struct udphdr)); */
282: /* XXXXX Check for from-one-location sockets, or from-any-location sockets */
283: ui->ui_src = saddr->sin_addr;
284: ui->ui_dst = daddr->sin_addr;
285: ui->ui_sport = saddr->sin_port;
286: ui->ui_dport = daddr->sin_port;
287: ui->ui_ulen = ui->ui_len;
288:
289: /*
290: * Stuff checksum and output datagram.
291: */
292: ui->ui_sum = 0;
293: if (udpcksum) {
294: if ((ui->ui_sum = cksum(m, /* sizeof (struct udpiphdr) + */ m->m_len)) == 0)
295: ui->ui_sum = 0xffff;
296: }
297: ((struct ip *)ui)->ip_len = m->m_len;
298:
299: ((struct ip *)ui)->ip_ttl = ip_defttl;
300: ((struct ip *)ui)->ip_tos = iptos;
301:
302: udpstat.udps_opackets++;
303:
304: error = ip_output(so, m);
305:
306: return (error);
307: }
308:
309: int udp_output(struct socket *so, struct mbuf *m,
310: struct sockaddr_in *addr)
311:
312: {
313: struct sockaddr_in saddr, daddr;
314:
315: saddr = *addr;
1.1.1.3 root 316: if ((so->so_faddr.s_addr & htonl(0xffffff00)) == special_addr.s_addr) {
1.1 root 317: saddr.sin_addr.s_addr = so->so_faddr.s_addr;
1.1.1.3 root 318: if ((so->so_faddr.s_addr & htonl(0x000000ff)) == htonl(0xff))
319: saddr.sin_addr.s_addr = alias_addr.s_addr;
320: }
1.1 root 321: daddr.sin_addr = so->so_laddr;
322: daddr.sin_port = so->so_lport;
323:
324: return udp_output2(so, m, &saddr, &daddr, so->so_iptos);
325: }
326:
327: int
328: udp_attach(so)
329: struct socket *so;
330: {
331: struct sockaddr_in addr;
332:
333: if((so->s = socket(AF_INET,SOCK_DGRAM,0)) != -1) {
334: /*
335: * Here, we bind() the socket. Although not really needed
336: * (sendto() on an unbound socket will bind it), it's done
337: * here so that emulation of ytalk etc. don't have to do it
338: */
339: addr.sin_family = AF_INET;
340: addr.sin_port = 0;
341: addr.sin_addr.s_addr = INADDR_ANY;
342: if(bind(so->s, (struct sockaddr *)&addr, sizeof(addr))<0) {
343: int lasterrno=errno;
344: closesocket(so->s);
345: so->s=-1;
346: #ifdef _WIN32
347: WSASetLastError(lasterrno);
348: #else
349: errno=lasterrno;
350: #endif
351: } else {
352: /* success, insert in queue */
353: so->so_expire = curtime + SO_EXPIRE;
354: insque(so,&udb);
355: }
356: }
357: return(so->s);
358: }
359:
360: void
361: udp_detach(so)
362: struct socket *so;
363: {
364: closesocket(so->s);
365: /* if (so->so_m) m_free(so->so_m); done by sofree */
366:
367: sofree(so);
368: }
369:
370: struct tos_t udptos[] = {
371: {0, 53, IPTOS_LOWDELAY, 0}, /* DNS */
372: {517, 517, IPTOS_LOWDELAY, EMU_TALK}, /* talk */
373: {518, 518, IPTOS_LOWDELAY, EMU_NTALK}, /* ntalk */
374: {0, 7648, IPTOS_LOWDELAY, EMU_CUSEEME}, /* Cu-Seeme */
375: {0, 0, 0, 0}
376: };
377:
378: u_int8_t
379: udp_tos(so)
380: struct socket *so;
381: {
382: int i = 0;
383:
384: while(udptos[i].tos) {
385: if ((udptos[i].fport && ntohs(so->so_fport) == udptos[i].fport) ||
386: (udptos[i].lport && ntohs(so->so_lport) == udptos[i].lport)) {
387: so->so_emu = udptos[i].emu;
388: return udptos[i].tos;
389: }
390: i++;
391: }
392:
393: return 0;
394: }
395:
396: #ifdef EMULATE_TALK
397: #include "talkd.h"
398: #endif
399:
400: /*
401: * Here, talk/ytalk/ntalk requests must be emulated
402: */
403: void
404: udp_emu(so, m)
405: struct socket *so;
406: struct mbuf *m;
407: {
408: struct sockaddr_in addr;
409: int addrlen = sizeof(addr);
410: #ifdef EMULATE_TALK
411: CTL_MSG_OLD *omsg;
412: CTL_MSG *nmsg;
413: char buff[sizeof(CTL_MSG)];
414: u_char type;
415:
416: struct talk_request {
417: struct talk_request *next;
418: struct socket *udp_so;
419: struct socket *tcp_so;
420: } *req;
421:
422: static struct talk_request *req_tbl = 0;
423:
424: #endif
425:
426: struct cu_header {
427: uint16_t d_family; // destination family
428: uint16_t d_port; // destination port
429: uint32_t d_addr; // destination address
430: uint16_t s_family; // source family
431: uint16_t s_port; // source port
432: uint32_t so_addr; // source address
433: uint32_t seqn; // sequence number
434: uint16_t message; // message
435: uint16_t data_type; // data type
436: uint16_t pkt_len; // packet length
437: } *cu_head;
438:
439: switch(so->so_emu) {
440:
441: #ifdef EMULATE_TALK
442: case EMU_TALK:
443: case EMU_NTALK:
444: /*
445: * Talk emulation. We always change the ctl_addr to get
446: * some answers from the daemon. When an ANNOUNCE comes,
447: * we send LEAVE_INVITE to the local daemons. Also when a
448: * DELETE comes, we send copies to the local daemons.
449: */
450: if (getsockname(so->s, (struct sockaddr *)&addr, &addrlen) < 0)
451: return;
452:
453: #define IS_OLD (so->so_emu == EMU_TALK)
454:
455: #define COPY_MSG(dest, src) { dest->type = src->type; \
456: dest->id_num = src->id_num; \
457: dest->pid = src->pid; \
458: dest->addr = src->addr; \
459: dest->ctl_addr = src->ctl_addr; \
460: memcpy(&dest->l_name, &src->l_name, NAME_SIZE_OLD); \
461: memcpy(&dest->r_name, &src->r_name, NAME_SIZE_OLD); \
462: memcpy(&dest->r_tty, &src->r_tty, TTY_SIZE); }
463:
464: #define OTOSIN(ptr, field) ((struct sockaddr_in *)&ptr->field)
465: /* old_sockaddr to sockaddr_in */
466:
467:
468: if (IS_OLD) { /* old talk */
469: omsg = mtod(m, CTL_MSG_OLD*);
470: nmsg = (CTL_MSG *) buff;
471: type = omsg->type;
472: OTOSIN(omsg, ctl_addr)->sin_port = addr.sin_port;
473: OTOSIN(omsg, ctl_addr)->sin_addr = our_addr;
474: strncpy(omsg->l_name, getlogin(), NAME_SIZE_OLD);
475: } else { /* new talk */
476: omsg = (CTL_MSG_OLD *) buff;
477: nmsg = mtod(m, CTL_MSG *);
478: type = nmsg->type;
479: OTOSIN(nmsg, ctl_addr)->sin_port = addr.sin_port;
480: OTOSIN(nmsg, ctl_addr)->sin_addr = our_addr;
481: strncpy(nmsg->l_name, getlogin(), NAME_SIZE_OLD);
482: }
483:
484: if (type == LOOK_UP)
485: return; /* for LOOK_UP this is enough */
486:
487: if (IS_OLD) { /* make a copy of the message */
488: COPY_MSG(nmsg, omsg);
489: nmsg->vers = 1;
490: nmsg->answer = 0;
491: } else
492: COPY_MSG(omsg, nmsg);
493:
494: /*
495: * If if is an ANNOUNCE message, we go through the
496: * request table to see if a tcp port has already
497: * been redirected for this socket. If not, we solisten()
498: * a new socket and add this entry to the table.
499: * The port number of the tcp socket and our IP
500: * are put to the addr field of the message structures.
501: * Then a LEAVE_INVITE is sent to both local daemon
502: * ports, 517 and 518. This is why we have two copies
503: * of the message, one in old talk and one in new talk
504: * format.
505: */
506:
507: if (type == ANNOUNCE) {
508: int s;
509: u_short temp_port;
510:
511: for(req = req_tbl; req; req = req->next)
512: if (so == req->udp_so)
513: break; /* found it */
514:
515: if (!req) { /* no entry for so, create new */
516: req = (struct talk_request *)
517: malloc(sizeof(struct talk_request));
518: req->udp_so = so;
519: req->tcp_so = solisten(0,
520: OTOSIN(omsg, addr)->sin_addr.s_addr,
521: OTOSIN(omsg, addr)->sin_port,
522: SS_FACCEPTONCE);
523: req->next = req_tbl;
524: req_tbl = req;
525: }
526:
527: /* replace port number in addr field */
528: addrlen = sizeof(addr);
529: getsockname(req->tcp_so->s,
530: (struct sockaddr *) &addr,
531: &addrlen);
532: OTOSIN(omsg, addr)->sin_port = addr.sin_port;
533: OTOSIN(omsg, addr)->sin_addr = our_addr;
534: OTOSIN(nmsg, addr)->sin_port = addr.sin_port;
535: OTOSIN(nmsg, addr)->sin_addr = our_addr;
536:
537: /* send LEAVE_INVITEs */
538: temp_port = OTOSIN(omsg, ctl_addr)->sin_port;
539: OTOSIN(omsg, ctl_addr)->sin_port = 0;
540: OTOSIN(nmsg, ctl_addr)->sin_port = 0;
541: omsg->type = nmsg->type = LEAVE_INVITE;
542:
543: s = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
544: addr.sin_addr = our_addr;
545: addr.sin_family = AF_INET;
546: addr.sin_port = htons(517);
547: sendto(s, (char *)omsg, sizeof(*omsg), 0,
548: (struct sockaddr *)&addr, sizeof(addr));
549: addr.sin_port = htons(518);
550: sendto(s, (char *)nmsg, sizeof(*nmsg), 0,
551: (struct sockaddr *) &addr, sizeof(addr));
552: closesocket(s) ;
553:
554: omsg->type = nmsg->type = ANNOUNCE;
555: OTOSIN(omsg, ctl_addr)->sin_port = temp_port;
556: OTOSIN(nmsg, ctl_addr)->sin_port = temp_port;
557: }
558:
559: /*
560: * If it is a DELETE message, we send a copy to the
561: * local daemons. Then we delete the entry corresponding
562: * to our socket from the request table.
563: */
564:
565: if (type == DELETE) {
566: struct talk_request *temp_req, *req_next;
567: int s;
568: u_short temp_port;
569:
570: temp_port = OTOSIN(omsg, ctl_addr)->sin_port;
571: OTOSIN(omsg, ctl_addr)->sin_port = 0;
572: OTOSIN(nmsg, ctl_addr)->sin_port = 0;
573:
574: s = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
575: addr.sin_addr = our_addr;
576: addr.sin_family = AF_INET;
577: addr.sin_port = htons(517);
578: sendto(s, (char *)omsg, sizeof(*omsg), 0,
579: (struct sockaddr *)&addr, sizeof(addr));
580: addr.sin_port = htons(518);
581: sendto(s, (char *)nmsg, sizeof(*nmsg), 0,
582: (struct sockaddr *)&addr, sizeof(addr));
583: closesocket(s);
584:
585: OTOSIN(omsg, ctl_addr)->sin_port = temp_port;
586: OTOSIN(nmsg, ctl_addr)->sin_port = temp_port;
587:
588: /* delete table entry */
589: if (so == req_tbl->udp_so) {
590: temp_req = req_tbl;
591: req_tbl = req_tbl->next;
592: free(temp_req);
593: } else {
594: temp_req = req_tbl;
595: for(req = req_tbl->next; req; req = req_next) {
596: req_next = req->next;
597: if (so == req->udp_so) {
598: temp_req->next = req_next;
599: free(req);
600: break;
601: } else {
602: temp_req = req;
603: }
604: }
605: }
606: }
607:
608: return;
609: #endif
610:
611: case EMU_CUSEEME:
612:
613: /*
614: * Cu-SeeMe emulation.
615: * Hopefully the packet is more that 16 bytes long. We don't
616: * do any other tests, just replace the address and port
617: * fields.
618: */
619: if (m->m_len >= sizeof (*cu_head)) {
620: if (getsockname(so->s, (struct sockaddr *)&addr, &addrlen) < 0)
621: return;
622: cu_head = mtod(m, struct cu_header *);
623: cu_head->s_port = addr.sin_port;
624: cu_head->so_addr = our_addr.s_addr;
625: }
626:
627: return;
628: }
629: }
630:
631: struct socket *
632: udp_listen(port, laddr, lport, flags)
633: u_int port;
634: u_int32_t laddr;
635: u_int lport;
636: int flags;
637: {
638: struct sockaddr_in addr;
639: struct socket *so;
640: int addrlen = sizeof(struct sockaddr_in), opt = 1;
641:
642: if ((so = socreate()) == NULL) {
643: free(so);
644: return NULL;
645: }
646: so->s = socket(AF_INET,SOCK_DGRAM,0);
647: so->so_expire = curtime + SO_EXPIRE;
648: insque(so,&udb);
649:
650: addr.sin_family = AF_INET;
651: addr.sin_addr.s_addr = INADDR_ANY;
652: addr.sin_port = port;
653:
654: if (bind(so->s,(struct sockaddr *)&addr, addrlen) < 0) {
655: udp_detach(so);
656: return NULL;
657: }
658: setsockopt(so->s,SOL_SOCKET,SO_REUSEADDR,(char *)&opt,sizeof(int));
659: /* setsockopt(so->s,SOL_SOCKET,SO_OOBINLINE,(char *)&opt,sizeof(int)); */
660:
661: getsockname(so->s,(struct sockaddr *)&addr,&addrlen);
662: so->so_fport = addr.sin_port;
663: if (addr.sin_addr.s_addr == 0 || addr.sin_addr.s_addr == loopback_addr.s_addr)
1.1.1.2 root 664: so->so_faddr = alias_addr;
1.1 root 665: else
666: so->so_faddr = addr.sin_addr;
667:
668: so->so_lport = lport;
669: so->so_laddr.s_addr = laddr;
670: if (flags != SS_FACCEPTONCE)
671: so->so_expire = 0;
672:
673: so->so_state = SS_ISFCONNECTED;
674:
675: return so;
676: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.