![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to cmds.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [CSRG BSD Unix] / 43BSD / usr.bin / tip|
Return to cmds.c CVS log | Up to [CSRG BSD Unix] / 43BSD / usr.bin / tip |
1.1 root 1: /*
2: * Copyright (c) 1983 Regents of the University of California.
3: * All rights reserved. The Berkeley software License Agreement
4: * specifies the terms and conditions for redistribution.
5: */
6:
7: #ifndef lint
8: static char sccsid[] = "@(#)cmds.c 5.4 (Berkeley) 5/5/86";
9: #endif not lint
10:
11: #include "tip.h"
12: /*
13: * tip
14: *
15: * miscellaneous commands
16: */
17:
18: int quant[] = { 60, 60, 24 };
19:
20: char null = '\0';
21: char *sep[] = { "second", "minute", "hour" };
22: static char *argv[10]; /* argument vector for take and put */
23:
24: int timeout(); /* timeout function called on alarm */
25: int stopsnd(); /* SIGINT handler during file transfers */
26: int intprompt(); /* used in handling SIG_INT during prompt */
27: int intcopy(); /* interrupt routine for file transfers */
28:
29: /*
30: * FTP - remote ==> local
31: * get a file from the remote host
32: */
33: getfl(c)
34: char c;
35: {
36: char buf[256], *cp, *expand();
37:
38: putchar(c);
39: /*
40: * get the UNIX receiving file's name
41: */
42: if (prompt("Local file name? ", copyname))
43: return;
44: cp = expand(copyname);
45: if ((sfd = creat(cp, 0666)) < 0) {
46: printf("\r\n%s: cannot creat\r\n", copyname);
47: return;
48: }
49:
50: /*
51: * collect parameters
52: */
53: if (prompt("List command for remote system? ", buf)) {
54: unlink(copyname);
55: return;
56: }
57: transfer(buf, sfd, value(EOFREAD));
58: }
59:
60: /*
61: * Cu-like take command
62: */
63: cu_take(cc)
64: char cc;
65: {
66: int fd, argc;
67: char line[BUFSIZ], *expand(), *cp;
68:
69: if (prompt("[take] ", copyname))
70: return;
71: if ((argc = args(copyname, argv)) < 1 || argc > 2) {
72: printf("usage: <take> from [to]\r\n");
73: return;
74: }
75: if (argc == 1)
76: argv[1] = argv[0];
77: cp = expand(argv[1]);
78: if ((fd = creat(cp, 0666)) < 0) {
79: printf("\r\n%s: cannot create\r\n", argv[1]);
80: return;
81: }
82: sprintf(line, "cat %s;echo \01", argv[0]);
83: transfer(line, fd, "\01");
84: }
85:
86: static jmp_buf intbuf;
87: /*
88: * Bulk transfer routine --
89: * used by getfl(), cu_take(), and pipefile()
90: */
91: transfer(buf, fd, eofchars)
92: char *buf, *eofchars;
93: {
94: register int ct;
95: char c, buffer[BUFSIZ];
96: register char *p = buffer;
97: register int cnt, eof;
98: time_t start;
99: int (*f)();
100:
101: pwrite(FD, buf, size(buf));
102: quit = 0;
103: kill(pid, SIGIOT);
104: read(repdes[0], (char *)&ccc, 1); /* Wait until read process stops */
105:
106: /*
107: * finish command
108: */
109: pwrite(FD, "\r", 1);
110: do
111: read(FD, &c, 1);
112: while ((c&0177) != '\n');
113: ioctl(0, TIOCSETC, &defchars);
114:
115: (void) setjmp(intbuf);
116: f = signal(SIGINT, intcopy);
117: start = time(0);
118: for (ct = 0; !quit;) {
119: eof = read(FD, &c, 1) <= 0;
120: c &= 0177;
121: if (quit)
122: continue;
123: if (eof || any(c, eofchars))
124: break;
125: if (c == 0)
126: continue; /* ignore nulls */
127: if (c == '\r')
128: continue;
129: *p++ = c;
130:
131: if (c == '\n' && boolean(value(VERBOSE)))
132: printf("\r%d", ++ct);
133: if ((cnt = (p-buffer)) == number(value(FRAMESIZE))) {
134: if (write(fd, buffer, cnt) != cnt) {
135: printf("\r\nwrite error\r\n");
136: quit = 1;
137: }
138: p = buffer;
139: }
140: }
141: if (cnt = (p-buffer))
142: if (write(fd, buffer, cnt) != cnt)
143: printf("\r\nwrite error\r\n");
144:
145: if (boolean(value(VERBOSE)))
146: prtime(" lines transferred in ", time(0)-start);
147: ioctl(0, TIOCSETC, &tchars);
148: write(fildes[1], (char *)&ccc, 1);
149: signal(SIGINT, f);
150: close(fd);
151: }
152:
153: /*
154: * FTP - remote ==> local process
155: * send remote input to local process via pipe
156: */
157: pipefile()
158: {
159: int cpid, pdes[2];
160: char buf[256];
161: int status, p;
162: extern int errno;
163:
164: if (prompt("Local command? ", buf))
165: return;
166:
167: if (pipe(pdes)) {
168: printf("can't establish pipe\r\n");
169: return;
170: }
171:
172: if ((cpid = fork()) < 0) {
173: printf("can't fork!\r\n");
174: return;
175: } else if (cpid) {
176: if (prompt("List command for remote system? ", buf)) {
177: close(pdes[0]), close(pdes[1]);
178: kill (cpid, SIGKILL);
179: } else {
180: close(pdes[0]);
181: signal(SIGPIPE, intcopy);
182: transfer(buf, pdes[1], value(EOFREAD));
183: signal(SIGPIPE, SIG_DFL);
184: while ((p = wait(&status)) > 0 && p != cpid)
185: ;
186: }
187: } else {
188: register int f;
189:
190: dup2(pdes[0], 0);
191: close(pdes[0]);
192: for (f = 3; f < 20; f++)
193: close(f);
194: execute(buf);
195: printf("can't execl!\r\n");
196: exit(0);
197: }
198: }
199:
200: /*
201: * Interrupt service routine for FTP
202: */
203: stopsnd()
204: {
205:
206: stop = 1;
207: signal(SIGINT, SIG_IGN);
208: }
209:
210: /*
211: * FTP - local ==> remote
212: * send local file to remote host
213: * terminate transmission with pseudo EOF sequence
214: */
215: sendfile(cc)
216: char cc;
217: {
218: FILE *fd;
219: char *fnamex;
220: char *expand();
221:
222: putchar(cc);
223: /*
224: * get file name
225: */
226: if (prompt("Local file name? ", fname))
227: return;
228:
229: /*
230: * look up file
231: */
232: fnamex = expand(fname);
233: if ((fd = fopen(fnamex, "r")) == NULL) {
234: printf("%s: cannot open\r\n", fname);
235: return;
236: }
237: transmit(fd, value(EOFWRITE), NULL);
238: if (!boolean(value(ECHOCHECK))) {
239: struct sgttyb buf;
240:
241: ioctl(FD, TIOCGETP, &buf); /* this does a */
242: ioctl(FD, TIOCSETP, &buf); /* wflushtty */
243: }
244: }
245:
246: /*
247: * Bulk transfer routine to remote host --
248: * used by sendfile() and cu_put()
249: */
250: transmit(fd, eofchars, command)
251: FILE *fd;
252: char *eofchars, *command;
253: {
254: char *pc, lastc;
255: int c, ccount, lcount;
256: time_t start_t, stop_t;
257: int (*f)();
258:
259: kill(pid, SIGIOT); /* put TIPOUT into a wait state */
260: stop = 0;
261: f = signal(SIGINT, stopsnd);
262: ioctl(0, TIOCSETC, &defchars);
263: read(repdes[0], (char *)&ccc, 1);
264: if (command != NULL) {
265: for (pc = command; *pc; pc++)
266: send(*pc);
267: if (boolean(value(ECHOCHECK)))
268: read(FD, (char *)&c, 1); /* trailing \n */
269: else {
270: struct sgttyb buf;
271:
272: ioctl(FD, TIOCGETP, &buf); /* this does a */
273: ioctl(FD, TIOCSETP, &buf); /* wflushtty */
274: sleep(5); /* wait for remote stty to take effect */
275: }
276: }
277: lcount = 0;
278: lastc = '\0';
279: start_t = time(0);
280: while (1) {
281: ccount = 0;
282: do {
283: c = getc(fd);
284: if (stop)
285: goto out;
286: if (c == EOF)
287: goto out;
288: if (c == 0177 && !boolean(value(RAWFTP)))
289: continue;
290: lastc = c;
291: if (c < 040) {
292: if (c == '\n') {
293: if (!boolean(value(RAWFTP)))
294: c = '\r';
295: }
296: else if (c == '\t') {
297: if (!boolean(value(RAWFTP))) {
298: if (boolean(value(TABEXPAND))) {
299: send(' ');
300: while ((++ccount % 8) != 0)
301: send(' ');
302: continue;
303: }
304: }
305: } else
306: if (!boolean(value(RAWFTP)))
307: continue;
308: }
309: send(c);
310: } while (c != '\r' && !boolean(value(RAWFTP)));
311: if (boolean(value(VERBOSE)))
312: printf("\r%d", ++lcount);
313: if (boolean(value(ECHOCHECK))) {
314: timedout = 0;
315: alarm(value(ETIMEOUT));
316: do { /* wait for prompt */
317: read(FD, (char *)&c, 1);
318: if (timedout || stop) {
319: if (timedout)
320: printf("\r\ntimed out at eol\r\n");
321: alarm(0);
322: goto out;
323: }
324: } while ((c&0177) != character(value(PROMPT)));
325: alarm(0);
326: }
327: }
328: out:
329: if (lastc != '\n' && !boolean(value(RAWFTP)))
330: send('\r');
331: for (pc = eofchars; *pc; pc++)
332: send(*pc);
333: stop_t = time(0);
334: fclose(fd);
335: signal(SIGINT, f);
336: if (boolean(value(VERBOSE)))
337: if (boolean(value(RAWFTP)))
338: prtime(" chars transferred in ", stop_t-start_t);
339: else
340: prtime(" lines transferred in ", stop_t-start_t);
341: write(fildes[1], (char *)&ccc, 1);
342: ioctl(0, TIOCSETC, &tchars);
343: }
344:
345: /*
346: * Cu-like put command
347: */
348: cu_put(cc)
349: char cc;
350: {
351: FILE *fd;
352: char line[BUFSIZ];
353: int argc;
354: char *expand();
355: char *copynamex;
356:
357: if (prompt("[put] ", copyname))
358: return;
359: if ((argc = args(copyname, argv)) < 1 || argc > 2) {
360: printf("usage: <put> from [to]\r\n");
361: return;
362: }
363: if (argc == 1)
364: argv[1] = argv[0];
365: copynamex = expand(argv[0]);
366: if ((fd = fopen(copynamex, "r")) == NULL) {
367: printf("%s: cannot open\r\n", copynamex);
368: return;
369: }
370: if (boolean(value(ECHOCHECK)))
371: sprintf(line, "cat>%s\r", argv[1]);
372: else
373: sprintf(line, "stty -echo;cat>%s;stty echo\r", argv[1]);
374: transmit(fd, "\04", line);
375: }
376:
377: /*
378: * FTP - send single character
379: * wait for echo & handle timeout
380: */
381: send(c)
382: char c;
383: {
384: char cc;
385: int retry = 0;
386:
387: cc = c;
388: pwrite(FD, &cc, 1);
389: #ifdef notdef
390: if (number(value(CDELAY)) > 0 && c != '\r')
391: nap(number(value(CDELAY)));
392: #endif
393: if (!boolean(value(ECHOCHECK))) {
394: #ifdef notdef
395: if (number(value(LDELAY)) > 0 && c == '\r')
396: nap(number(value(LDELAY)));
397: #endif
398: return;
399: }
400: tryagain:
401: timedout = 0;
402: alarm(value(ETIMEOUT));
403: read(FD, &cc, 1);
404: alarm(0);
405: if (timedout) {
406: printf("\r\ntimeout error (%s)\r\n", ctrl(c));
407: if (retry++ > 3)
408: return;
409: pwrite(FD, &null, 1); /* poke it */
410: goto tryagain;
411: }
412: }
413:
414: timeout()
415: {
416: signal(SIGALRM, timeout);
417: timedout = 1;
418: }
419:
420: /*
421: * Stolen from consh() -- puts a remote file on the output of a local command.
422: * Identical to consh() except for where stdout goes.
423: */
424: pipeout(c)
425: {
426: char buf[256];
427: int cpid, status, p;
428: time_t start;
429:
430: putchar(c);
431: if (prompt("Local command? ", buf))
432: return;
433: kill(pid, SIGIOT); /* put TIPOUT into a wait state */
434: signal(SIGINT, SIG_IGN);
435: signal(SIGQUIT, SIG_IGN);
436: ioctl(0, TIOCSETC, &defchars);
437: read(repdes[0], (char *)&ccc, 1);
438: /*
439: * Set up file descriptors in the child and
440: * let it go...
441: */
442: if ((cpid = fork()) < 0)
443: printf("can't fork!\r\n");
444: else if (cpid) {
445: start = time(0);
446: while ((p = wait(&status)) > 0 && p != cpid)
447: ;
448: } else {
449: register int i;
450:
451: dup2(FD, 1);
452: for (i = 3; i < 20; i++)
453: close(i);
454: signal(SIGINT, SIG_DFL);
455: signal(SIGQUIT, SIG_DFL);
456: execute(buf);
457: printf("can't find `%s'\r\n", buf);
458: exit(0);
459: }
460: if (boolean(value(VERBOSE)))
461: prtime("away for ", time(0)-start);
462: write(fildes[1], (char *)&ccc, 1);
463: ioctl(0, TIOCSETC, &tchars);
464: signal(SIGINT, SIG_DFL);
465: signal(SIGQUIT, SIG_DFL);
466: }
467:
468: #ifdef CONNECT
469: /*
470: * Fork a program with:
471: * 0 <-> local tty in
472: * 1 <-> local tty out
473: * 2 <-> local tty out
474: * 3 <-> remote tty in
475: * 4 <-> remote tty out
476: */
477: consh(c)
478: {
479: char buf[256];
480: int cpid, status, p;
481: time_t start;
482:
483: putchar(c);
484: if (prompt("Local command? ", buf))
485: return;
486: kill(pid, SIGIOT); /* put TIPOUT into a wait state */
487: signal(SIGINT, SIG_IGN);
488: signal(SIGQUIT, SIG_IGN);
489: ioctl(0, TIOCSETC, &defchars);
490: read(repdes[0], (char *)&ccc, 1);
491: /*
492: * Set up file descriptors in the child and
493: * let it go...
494: */
495: if ((cpid = fork()) < 0)
496: printf("can't fork!\r\n");
497: else if (cpid) {
498: start = time(0);
499: while ((p = wait(&status)) > 0 && p != cpid)
500: ;
501: } else {
502: register int i;
503:
504: dup2(FD, 3);
505: dup2(3, 4);
506: for (i = 5; i < 20; i++)
507: close(i);
508: signal(SIGINT, SIG_DFL);
509: signal(SIGQUIT, SIG_DFL);
510: execute(buf);
511: printf("can't find `%s'\r\n", buf);
512: exit(0);
513: }
514: if (boolean(value(VERBOSE)))
515: prtime("away for ", time(0)-start);
516: write(fildes[1], (char *)&ccc, 1);
517: ioctl(0, TIOCSETC, &tchars);
518: signal(SIGINT, SIG_DFL);
519: signal(SIGQUIT, SIG_DFL);
520: }
521: #endif
522:
523: /*
524: * Escape to local shell
525: */
526: shell()
527: {
528: int shpid, status;
529: extern char **environ;
530: char *cp;
531:
532: printf("[sh]\r\n");
533: signal(SIGINT, SIG_IGN);
534: signal(SIGQUIT, SIG_IGN);
535: unraw();
536: if (shpid = fork()) {
537: while (shpid != wait(&status));
538: raw();
539: printf("\r\n!\r\n");
540: signal(SIGINT, SIG_DFL);
541: signal(SIGQUIT, SIG_DFL);
542: return;
543: } else {
544: signal(SIGQUIT, SIG_DFL);
545: signal(SIGINT, SIG_DFL);
546: if ((cp = rindex(value(SHELL), '/')) == NULL)
547: cp = value(SHELL);
548: else
549: cp++;
550: execl(value(SHELL), cp, 0);
551: printf("\r\ncan't execl!\r\n");
552: exit(1);
553: }
554: }
555:
556: /*
557: * TIPIN portion of scripting
558: * initiate the conversation with TIPOUT
559: */
560: setscript()
561: {
562: char c;
563: /*
564: * enable TIPOUT side for dialogue
565: */
566: kill(pid, SIGEMT);
567: if (boolean(value(SCRIPT)))
568: write(fildes[1], value(RECORD), size(value(RECORD)));
569: write(fildes[1], "\n", 1);
570: /*
571: * wait for TIPOUT to finish
572: */
573: read(repdes[0], &c, 1);
574: if (c == 'n')
575: printf("can't create %s\r\n", value(RECORD));
576: }
577:
578: /*
579: * Change current working directory of
580: * local portion of tip
581: */
582: chdirectory()
583: {
584: char dirname[80];
585: register char *cp = dirname;
586:
587: if (prompt("[cd] ", dirname)) {
588: if (stoprompt)
589: return;
590: cp = value(HOME);
591: }
592: if (chdir(cp) < 0)
593: printf("%s: bad directory\r\n", cp);
594: printf("!\r\n");
595: }
596:
597: abort(msg)
598: char *msg;
599: {
600:
601: kill(pid, SIGTERM);
602: setreuid(euid, euid);
603: setregid(egid, egid);
604: disconnect(msg);
605: if (msg != NOSTR)
606: printf("\r\n%s", msg);
607: printf("\r\n[EOT]\r\n");
608: delock(uucplock);
609: unraw();
610: exit(0);
611: }
612:
613: finish()
614: {
615: char *dismsg;
616:
617: if ((dismsg = value(DISCONNECT)) != NOSTR) {
618: write(FD, dismsg, strlen(dismsg));
619: sleep(5);
620: }
621: abort(NOSTR);
622: }
623:
624: intcopy()
625: {
626:
627: raw();
628: quit = 1;
629: longjmp(intbuf, 1);
630: }
631:
632: execute(s)
633: char *s;
634: {
635: register char *cp;
636:
637: if ((cp = rindex(value(SHELL), '/')) == NULL)
638: cp = value(SHELL);
639: else
640: cp++;
641: execl(value(SHELL), cp, "-c", s, 0);
642: }
643:
644: args(buf, a)
645: char *buf, *a[];
646: {
647: register char *p = buf, *start;
648: register char **parg = a;
649: register int n = 0;
650:
651: do {
652: while (*p && (*p == ' ' || *p == '\t'))
653: p++;
654: start = p;
655: if (*p)
656: *parg = p;
657: while (*p && (*p != ' ' && *p != '\t'))
658: p++;
659: if (p != start)
660: parg++, n++;
661: if (*p)
662: *p++ = '\0';
663: } while (*p);
664:
665: return(n);
666: }
667:
668: prtime(s, a)
669: char *s;
670: time_t a;
671: {
672: register i;
673: int nums[3];
674:
675: for (i = 0; i < 3; i++) {
676: nums[i] = (int)(a % quant[i]);
677: a /= quant[i];
678: }
679: printf("%s", s);
680: while (--i >= 0)
681: if (nums[i] || i == 0 && nums[1] == 0 && nums[2] == 0)
682: printf("%d %s%c ", nums[i], sep[i],
683: nums[i] == 1 ? '\0' : 's');
684: printf("\r\n!\r\n");
685: }
686:
687: variable()
688: {
689: char buf[256];
690:
691: if (prompt("[set] ", buf))
692: return;
693: vlex(buf);
694: if (vtable[BEAUTIFY].v_access&CHANGED) {
695: vtable[BEAUTIFY].v_access &= ~CHANGED;
696: kill(pid, SIGSYS);
697: }
698: if (vtable[SCRIPT].v_access&CHANGED) {
699: vtable[SCRIPT].v_access &= ~CHANGED;
700: setscript();
701: /*
702: * So that "set record=blah script" doesn't
703: * cause two transactions to occur.
704: */
705: if (vtable[RECORD].v_access&CHANGED)
706: vtable[RECORD].v_access &= ~CHANGED;
707: }
708: if (vtable[RECORD].v_access&CHANGED) {
709: vtable[RECORD].v_access &= ~CHANGED;
710: if (boolean(value(SCRIPT)))
711: setscript();
712: }
713: if (vtable[TAND].v_access&CHANGED) {
714: vtable[TAND].v_access &= ~CHANGED;
715: if (boolean(value(TAND)))
716: tandem("on");
717: else
718: tandem("off");
719: }
720: if (vtable[LECHO].v_access&CHANGED) {
721: vtable[LECHO].v_access &= ~CHANGED;
722: HD = boolean(value(LECHO));
723: }
724: if (vtable[PARITY].v_access&CHANGED) {
725: vtable[PARITY].v_access &= ~CHANGED;
726: setparity();
727: }
728: }
729:
730: /*
731: * Turn tandem mode on or off for remote tty.
732: */
733: tandem(option)
734: char *option;
735: {
736: struct sgttyb rmtty;
737:
738: ioctl(FD, TIOCGETP, &rmtty);
739: if (strcmp(option,"on") == 0) {
740: rmtty.sg_flags |= TANDEM;
741: arg.sg_flags |= TANDEM;
742: } else {
743: rmtty.sg_flags &= ~TANDEM;
744: arg.sg_flags &= ~TANDEM;
745: }
746: ioctl(FD, TIOCSETP, &rmtty);
747: ioctl(0, TIOCSETP, &arg);
748: }
749:
750: /*
751: * Send a break.
752: */
753: genbrk()
754: {
755:
756: ioctl(FD, TIOCSBRK, NULL);
757: sleep(1);
758: ioctl(FD, TIOCCBRK, NULL);
759: }
760:
761: /*
762: * Suspend tip
763: */
764: suspend(c)
765: char c;
766: {
767:
768: unraw();
769: kill(c == CTRL(y) ? getpid() : 0, SIGTSTP);
770: raw();
771: }
772:
773: /*
774: * expand a file name if it includes shell meta characters
775: */
776:
777: char *
778: expand(name)
779: char name[];
780: {
781: static char xname[BUFSIZ];
782: char cmdbuf[BUFSIZ];
783: register int pid, l, rc;
784: register char *cp, *Shell;
785: int s, pivec[2], (*sigint)();
786:
787: if (!anyof(name, "~{[*?$`'\"\\"))
788: return(name);
789: /* sigint = signal(SIGINT, SIG_IGN); */
790: if (pipe(pivec) < 0) {
791: perror("pipe");
792: /* signal(SIGINT, sigint) */
793: return(name);
794: }
795: sprintf(cmdbuf, "echo %s", name);
796: if ((pid = vfork()) == 0) {
797: Shell = value(SHELL);
798: if (Shell == NOSTR)
799: Shell = "/bin/sh";
800: close(pivec[0]);
801: close(1);
802: dup(pivec[1]);
803: close(pivec[1]);
804: close(2);
805: execl(Shell, Shell, "-c", cmdbuf, 0);
806: _exit(1);
807: }
808: if (pid == -1) {
809: perror("fork");
810: close(pivec[0]);
811: close(pivec[1]);
812: return(NOSTR);
813: }
814: close(pivec[1]);
815: l = read(pivec[0], xname, BUFSIZ);
816: close(pivec[0]);
817: while (wait(&s) != pid);
818: ;
819: s &= 0377;
820: if (s != 0 && s != SIGPIPE) {
821: fprintf(stderr, "\"Echo\" failed\n");
822: return(NOSTR);
823: }
824: if (l < 0) {
825: perror("read");
826: return(NOSTR);
827: }
828: if (l == 0) {
829: fprintf(stderr, "\"%s\": No match\n", name);
830: return(NOSTR);
831: }
832: if (l == BUFSIZ) {
833: fprintf(stderr, "Buffer overflow expanding \"%s\"\n", name);
834: return(NOSTR);
835: }
836: xname[l] = 0;
837: for (cp = &xname[l-1]; *cp == '\n' && cp > xname; cp--)
838: ;
839: *++cp = '\0';
840: return(xname);
841: }
842:
843: /*
844: * Are any of the characters in the two strings the same?
845: */
846:
847: anyof(s1, s2)
848: register char *s1, *s2;
849: {
850: register int c;
851:
852: while (c = *s1++)
853: if (any(c, s2))
854: return(1);
855: return(0);
856: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.