![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to funcs2.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [CSRG BSD Unix] / 43BSDTahoe / new / news / src|
Return to funcs2.c CVS log | Up to [CSRG BSD Unix] / 43BSDTahoe / new / news / src |
1.1 root 1: /*
2: * This software is Copyright (c) 1985 by Rick Adams.
3: *
4: * Permission is hereby granted to copy, reproduce, redistribute or
5: * otherwise use this software as long as: there is no monetary
6: * profit gained specifically from the use or reproduction or this
7: * software, it is not sold, rented, traded or otherwise marketed, and
8: * this copyright notice is included prominently in any copy
9: * made.
10: *
11: * The author make no claims as to the fitness or correctness of
12: * this software for any use whatsoever, and it is provided as is.
13: * Any use of this software is at the user's own risk.
14: *
15: *
16: * funcs2 - functions used by both inews and readnews.
17: */
18:
19: #ifdef SCCSID
20: static char *SccsId = "@(#)funcs2.c 1.22 10/15/87";
21: #endif /* SCCSID */
22:
23: #include "params.h"
24:
25: #ifdef SunIII
26: #ifndef INTERNET
27: #define INTERNET
28: #endif /* !INTERNET */
29: #endif /* SunIII */
30:
31: /*LINTLIBRARY*/
32:
33: /*
34: * Get user name and home directory.
35: */
36: getuser()
37: {
38: static int flag = TRUE;
39: register struct passwd *p;
40:
41: if (flag) {
42: if ((p = getpwuid(uid)) == NULL)
43: xerror("Cannot get user's name");
44: if ( username == NULL || username[0] == 0 ||
45: STRCMP(username, "Unknown") == 0)
46: username = AllocCpy(p->pw_name);
47: userhome = AllocCpy(p->pw_dir);
48: flag = FALSE;
49: }
50: (void) strcpy(header.path, username);
51: }
52:
53: /* no sys file on clients via nntp */
54: #ifndef SERVER
55: static FILE *sysfile;
56:
57: char *fldget();
58:
59: static int sfline;
60:
61: /*
62: * Open SUBFILE.
63: */
64: s_openr()
65: {
66: sysfile = xfopen(SUBFILE, "r");
67: sfline = 0;
68: }
69:
70: /*
71: * Read SUBFILE.
72: */
73: s_read(sp)
74: register struct srec *sp;
75: {
76: register char *p;
77: register int c;
78: char *e;
79: int chop_spaces = 0;
80: again:
81: p = bfr;
82: /*
83: * Read the SUBFILE (/usr/lib/news/sys) from the current
84: * position to the first unescaped newline. If a newline is
85: * escaped with a backslash (\) continue reading but throw away
86: * the backslash and newline; read the next line skipping spaces
87: * and tabs until the first non-space/tab character, then start
88: * looking for a newline again. Skipping the leading
89: * spaces/tabs after a escaped newline keeps the news groups
90: * together. If a line begins with a newline, just skip it.
91: */
92: for (e=p+LBUFLEN; p < e && (c=getc(sysfile)) != EOF; p++) {
93: *p = c;
94: if (c == '\n') {
95: sfline++;
96: if (p == bfr || p[-1] != '\\') {
97: p[1] = '\0';
98: break;
99: } else {
100: chop_spaces++;
101: p -= 2;
102: }
103: } else if (chop_spaces) {
104: if (c == '\t' || c == ' ')
105: p--;
106: else
107: chop_spaces = 0;
108: }
109: }
110: if (c == EOF) {
111: return FALSE;
112: }
113: p = bfr;
114: while (*p == ' ' || *p == '\t') /* skip leading white space */
115: p++;
116: if (*p == '\n')
117: goto again; /* skip newlines */
118: if (!nstrip(p))
119: xerror("SUBFILE (%s) line %d too long.", SUBFILE, sfline);
120: if (*p == '#')
121: goto again;
122: sp->s_xmit[0] = '\0';
123: sp->s_flags[0] = '\0';
124: sp->s_nosend = (char *)0;
125:
126: p = fldget(sp->s_name, p);
127: if (*p++ == '\0')
128: xerror("Bad SUBFILE (%s) line %d.", SUBFILE, sfline);
129: /*
130: * A sys file line reading "ME" means the name of the local system.
131: */
132: if (STRCMP(sp->s_name, "ME") == 0)
133: (void) strcpy(sp->s_name, LOCALPATHSYSNAME);
134: e = index(sp->s_name, '/');
135: if (e) {
136: *e++ = '\0';
137: sp->s_nosend = e;
138: }
139: p = fldget(sp->s_nbuf, p);
140: lcase(sp->s_nbuf);
141: if (*p++ == '\0')
142: return TRUE;
143:
144: p = fldget(sp->s_flags, p);
145: if (*p++ == '\0')
146: return TRUE;
147:
148: (void) fldget(sp->s_xmit, p);
149: return TRUE;
150: }
151:
152: char *
153: fldget(q, p)
154: register char *q, *p;
155: {
156: while (*p && *p != ':') {
157: if (*p == '\\' && p[1]==':')
158: p++;
159: *q++ = *p++;
160: }
161: *q = '\0';
162: return p;
163: }
164:
165: /*
166: * Find the SUBFILE record for a system.
167: */
168: s_find(sp, system)
169: register struct srec *sp;
170: char *system;
171: {
172: s_openr();
173: while (s_read(sp))
174: if (STRNCMP(system, sp->s_name, SNLN) == 0) {
175: s_close();
176: return TRUE;
177: }
178: s_close();
179: return FALSE;
180: }
181:
182: /*
183: * Close sysfile.
184: */
185: s_close()
186: {
187: (void) fclose(sysfile);
188: }
189: #endif /* SERVER */
190:
191: extern struct timeb Now;
192:
193: time_t
194: cgtdate(datestr)
195: char *datestr;
196: {
197: char junk[40],month[40],day[30],tod[60],year[50];
198: static time_t lasttime;
199: static char lastdatestr[BUFLEN] = "";
200:
201: if ( lastdatestr[0] && STRCMP(datestr, lastdatestr) == 0)
202: return lasttime;
203: lasttime = getdate(datestr, &Now);
204: if (lasttime < 0 &&
205: sscanf(datestr, "%s %s %s %s %s", junk, month, day, tod, year) == 5) {
206: (void) sprintf(bfr, "%s %s, %s %s", month, day, year, tod);
207: lasttime = getdate(bfr, &Now);
208: if (lasttime < 0) {
209: logerr("Unparsable date \"%s\"", datestr);
210: datestr = "now"; /* better than nothing */
211: lasttime = Now.time;
212: }
213: }
214: strncpy(lastdatestr, datestr, BUFLEN);
215: return lasttime;
216: }
217:
218: lcase(s)
219: register char *s;
220: {
221: register char *ptr;
222:
223: for (ptr = s; *ptr; ptr++)
224: if (isupper(*ptr))
225: *ptr = tolower(*ptr);
226: }
227:
228: /*
229: * Return a compact representation of the person who posted the given
230: * message. A sender or internet name will be used, otherwise
231: * the last part of the path is used preceded by an optional ".."
232: */
233: char *
234: tailpath(hp)
235: struct hbuf *hp;
236: {
237: char *p, *r;
238: static char resultbuf[BUFLEN];
239: char pathbuf[PATHLEN];
240: char *malloc();
241:
242: /*
243: * This only happens for articles posted by old news software
244: * in non-internet format.
245: */
246: resultbuf[0] = '\0';
247: (void) strncpy(pathbuf, hp->path, PATHLEN);
248: p = index(pathbuf, ' ');
249: if (p)
250: *p = '\0'; /* Chop off trailing " (name)" */
251: r = rindex(pathbuf, '!');
252: if (r == 0) {
253: r = pathbuf;
254: } else {
255: while (r > pathbuf && *--r != '!')
256: ;
257: if (r > pathbuf) {
258: r++;
259: (void) strcpy(resultbuf, "..!");
260: }
261: }
262: (void) strcat(resultbuf, r);
263: return resultbuf;
264: }
265:
266: /*
267: * arpadate is like ctime(3) except that the time is returned in
268: * an acceptable ARPANET time format instead of ctime format.
269: */
270: char *
271: arpadate(longtime)
272: time_t *longtime;
273: {
274: register char *p, *q, *ud;
275: register int i;
276: static char b[40];
277: extern struct tm *gmtime();
278: extern char *asctime();
279:
280: /* Get current time. This will be used resolve the timezone. */
281: ud = asctime(gmtime(longtime));
282:
283: /* Crack the UNIX date line in a singularly unoriginal way. */
284: q = b;
285:
286: #ifdef notdef
287: /* until every site installs the fix to getdate.y, the day
288: of the week can cause time warps */
289: p = &ud[0]; /* Mon */
290: *q++ = *p++;
291: *q++ = *p++;
292: *q++ = *p++;
293: *q++ = ','; *q++ = ' ';
294: #endif
295:
296: p = &ud[8]; /* 16 */
297: if (*p == ' ')
298: p++;
299: else
300: *q++ = *p++;
301: *q++ = *p++; *q++ = ' ';
302:
303: p = &ud[4]; /* Sep */
304: *q++ = *p++; *q++ = *p++; *q++ = *p++; *q++ = ' ';
305:
306: p = &ud[22]; /* 1979 */
307: *q++ = *p++; *q++ = *p++; *q++ = ' ';
308:
309: p = &ud[11]; /* 01:03:52 */
310: for (i = 8; i > 0; i--)
311: *q++ = *p++;
312:
313: *q++ = ' ';
314: *q++ = 'G'; /* GMT */
315: *q++ = 'M';
316: *q++ = 'T';
317: *q = '\0';
318:
319: return b;
320: }
321:
322: char *
323: replyname(hptr)
324: struct hbuf *hptr;
325: {
326: register char *ptr;
327: static char tbuf[PATHLEN];
328:
329: ptr = hptr->path;
330: if (PREFIX(ptr, PATHSYSNAME) &&
331: index(NETCHRS, ptr[strlen(PATHSYSNAME)]))
332: ptr = index(ptr, '!') + 1;
333: #ifdef INTERNET
334: if (hptr->from[0])
335: ptr = hptr->from;
336: if (hptr->replyto[0])
337: ptr = hptr->replyto;
338: #else /* !INTERNET */
339: if (hptr->replyto[0] && !index(hptr->replyto, '@'))
340: ptr = hptr->replyto;
341: #endif
342: (void) strcpy(tbuf, ptr);
343: ptr = index(tbuf, '(');
344: if (ptr) {
345: while (ptr[-1] == ' ')
346: ptr--;
347: *ptr = 0;
348: }
349: #ifdef SunIII
350: if (ptr = rindex(tbuf, '.')) {
351: if (PREFIX(++ptr, "OZ")) {
352: /* some people only allow it in lower case ... */
353: strcpy(ptr, "oz");
354: return tbuf;
355: }
356: if (PREFIX(ptr, "UUCP") || PREFIX(ptr, "ARPA") ||
357: PREFIX(ptr, "DEC") || PREFIX(ptr, "CSNET")) {
358: strcat(tbuf, "@munnari.oz"); /* via sun to munnari */
359: return tbuf;
360: }
361: }
362: /*
363: * must(?) have come from a uucp site, lets look see if path passes
364: * through munnari, and if so delete the fake uucp path after that.
365: */
366: for (ptr = tbuf ;; ptr++) {
367: if (PREFIX(ptr, "munnari!")) {
368: strcpy(tbuf, ptr+8);
369: break;
370: }
371: ptr = index(ptr, '!');
372: if (ptr == (char *)0)
373: break;
374: }
375: /*
376: * now, just send the address we have left to munnari, and
377: * hope that something sensible will be done with it there.
378: * (This works in more cases than you'd think ...)
379: */
380: strcat(tbuf, "@munnari.oz");
381: #else /* !SunIII */
382: #ifndef INTERNET
383: /*
384: * Play games stripping off multiple berknet
385: * addresses (a!b!c:d:e => a!b!d:e) here.
386: */
387: for (ptr=tbuf; *ptr; ptr++) {
388: register char *ptr2;
389:
390: if (index(NETCHRS, *ptr) && *ptr == ':' &&
391: (ptr2=index(ptr+1, ':')))
392: (void) strcpy(ptr, ptr2);
393: }
394: #else /* INTERNET */
395: {
396: char mbuf[BUFLEN], modadd[BUFLEN];
397: FILE *mfd;
398: /* Let's find a path to the backbone */
399: sprintf(mbuf, "%s/mailpaths", LIB);
400: mfd = xfopen(mbuf, "r");
401: do {
402: if (fgets(mbuf, sizeof mbuf, mfd) == NULL)
403: xerror("Can't find internet in %s/mailpaths",
404: LIB);
405: } while (!PREFIX(mbuf, "internet"));
406: if (sscanf(mbuf, "%*s %s", modadd) != 1)
407: xerror("backbone address corrupted");
408: (void) fclose(mfd);
409: (void)strcpy(mbuf, tbuf);
410: /* If we are lucky, there is no ! or @ in the forward address */
411: if (strpbrk(modadd, "!@") == NULL) {
412: sprintf(tbuf, modadd, mbuf);
413: } else {
414: char *cp = index(mbuf, '@');
415: if (index(modadd, '@') == NULL && cp) {
416: /* we have to rearrange the address so no @ are in it */
417: char atbuf[BUFLEN];
418: *cp++ = '\0';
419: sprintf(atbuf, "%s!%s", cp, mbuf);
420: sprintf(tbuf, modadd, atbuf);
421: } else if (cp) {
422: /* some days you don't get lucky. presume the % hack */
423: *cp = '%';
424: sprintf(tbuf, modadd, mbuf);
425: }
426: }
427: }
428: #endif /* INTERNET */
429: #endif /* !SunIII */
430: return tbuf;
431: }
432:
433:
434: /*
435: * Given an article ID, find the line in the history file that mentions it.
436: * Return the text of the line, or NULL if not found. A pointer to a
437: * static area is returned.
438: */
439: char *
440: findhist(artid)
441: char *artid;
442: {
443: static char lbuf[256];
444: char oidbuf[BUFSIZ];
445: FILE *hfp;
446: register char *p;
447: #ifdef SERVER
448: char workspace[256];
449: struct tm *tm;
450: long clock;
451: #else /* !SERVER */
452: #ifdef DBM
453: datum lhs, rhs;
454: datum fetch();
455: long fpos; /* We have to use an explicit variable to insure alignment */
456: #else /* !DBM */
457: char *histfile();
458: #endif /* !DBM */
459: #endif /* !SERVER */
460: /* Try to understand old artid's as well. Assume .UUCP domain. */
461: if (artid[0] != '<') {
462: p = index(artid, '.');
463: if (p)
464: *p++ = '\0';
465: (void) sprintf(oidbuf, "<%s@%s.UUCP>", p, artid);
466: if (p)
467: *--p = '.';
468: } else
469: (void) strcpy(oidbuf, artid);
470: #ifdef SERVER
471: (void) sprintf(lbuf,"STAT %s",oidbuf);
472: put_server(lbuf);
473: (void) get_server(workspace,sizeof(workspace));
474: if (*workspace != CHAR_OK)
475: return NULL;
476: (void) sprintf(lbuf,"XHDR xref %s",oidbuf);
477: put_server(lbuf);
478: (void) get_server(workspace,sizeof(workspace)); /* get response */
479: if (*workspace != CHAR_OK)
480: return NULL; /* old style nntp */
481: (void) get_server(workspace,sizeof(workspace)); /* get header line */
482: sync_server(); /* get rid of the rest of it */
483: p = index(workspace,' ');
484: p++;
485:
486: if (*p == '(') { /* there is no xref line */
487: long s,sm;
488: FILE * af;
489: char n[100], buf[100], *name;
490: (void) sprintf(lbuf,"XHDR newsgroups %s",oidbuf);
491: put_server(lbuf);
492: (void) get_server(workspace,sizeof(workspace));
493: if (*workspace != CHAR_OK)
494: return NULL;
495: (void) get_server(workspace,sizeof(workspace));
496: sync_server();
497: if ((name = index(workspace,' ')) == NULL)
498: return NULL;
499: name++;
500: /* now we fetch the line from the active file */
501: af = xfopen(ACTIVE, "r");
502: while (fgets(buf, sizeof(buf), af) != NULL) {
503: if (sscanf(buf, "%s %ld %ld", n, &s, &sm) == 3 &&
504: STRCMP(n, name) == 0) {
505: break;
506: }
507: }
508: (void) fclose(af);
509: /* now we ask for a message ids in that newsgroup */
510: if (set_group(name) == NULL)
511: return NULL;
512: (void) sprintf(lbuf, "XHDR message-id %d-%d", sm, s);
513: put_server(lbuf);
514: (void) get_server(workspace,sizeof(workspace));
515: if (*workspace != CHAR_OK)
516: return NULL;
517: while ( get_server(workspace,sizeof(workspace)) >= 0) {
518: if (*workspace == '.' && strlen(workspace) == 1)
519: return NULL;
520: if (strindex(workspace,oidbuf) > -1)
521: break;
522: }
523: sync_server();
524: *(index(workspace,' ')) = '\0';
525: (void) sprintf(lbuf, "%s/%s", n, workspace);
526: bzero(workspace,sizeof(workspace));
527: strcpy(workspace, lbuf);
528: } else {
529: bzero(lbuf, sizeof(lbuf));
530: strcpy(lbuf, p);
531: while (*p != '\0' && (p = index(lbuf,':')) != NULL) {
532: *p = '/';
533: p++;
534: }
535: strcpy(workspace, lbuf);
536: }
537: p = &workspace[0];
538: time(&clock);
539: tm = localtime(&clock);
540: #ifdef USG
541: sprintf(lbuf, "%s\t%2.2d/%2.2d/%d %2.2d:%2.2d\t%s",
542: #else /* !USG */
543: sprintf(lbuf, "%s\t%02d/%02d/%d %02d:%02d\t%s",
544: #endif /* !USG */
545: oidbuf,tm->tm_mon,tm->tm_mday,tm->tm_year,tm->tm_hour,tm->tm_min,p);
546: return lbuf; /* not really the same, but close */
547: #else /* !SERVER */
548: lcase(oidbuf);
549: #ifdef DBM
550: initdbm(ARTFILE);
551: lhs.dptr = oidbuf;
552: lhs.dsize = strlen(lhs.dptr) + 1;
553: rhs = fetch(lhs);
554: if (rhs.dptr == NULL)
555: return NULL;
556: hfp = xfopen(ARTFILE, "r");
557: /* The bcopy is NECESSARY to insure alignment on some machines */
558: bcopy(rhs.dptr, (char *)&fpos, sizeof (long));
559: fseek(hfp, fpos, 0);
560: #else /* !DBM */
561: hfp = xfopen(histfile(oidbuf), "r");
562: #endif /* !DBM */
563: while (fgets(lbuf, BUFLEN, hfp) != NULL) {
564: p = index(lbuf, '\t');
565: if (p == NULL)
566: p = index(lbuf, '\n');
567: *p = 0;
568: if (STRCMP(lbuf, artid) == 0 || STRCMP(lbuf, oidbuf) == 0) {
569: (void) fclose(hfp);
570: *p = '\t';
571: *(lbuf + strlen(lbuf) - 1) = 0; /* zap the \n */
572: return lbuf;
573: }
574: #ifdef DBM
575: break;
576: #endif /* DBM */
577: }
578: (void) fclose(hfp);
579: return NULL;
580: #endif /* !SERVER */
581: }
582:
583: /*
584: * Hunt up the article "artid", and return the newsgroup/artnum
585: * where it can be found.
586: */
587: char *
588: findfname(artid)
589: char *artid;
590: {
591: char *line, *p, *q;
592: char *findhist();
593: static char fname[BUFLEN];
594:
595: line = findhist(artid);
596: if (line) {
597: /* Look for it stored as an article, where it should be */
598: p = index(line, '\t');
599: p = index(p+1, '\t');
600: p++;
601: if (*p) {
602: q = index(p, ' ');
603: if (q)
604: *q = 0;
605: (void) strcpy(fname, p);
606: return fname;
607: }
608: }
609: return NULL;
610: }
611:
612: /*
613: * Hunt up the article "artid", fopen it for read, and return a
614: * file descriptor to it. We look everywhere we can think of.
615: */
616: FILE *
617: hfopen(artid)
618: char *artid;
619: {
620: char *p;
621: char *findhist();
622: FILE *rv = NULL;
623: char fname[BUFLEN];
624:
625: p = findfname(artid);
626: if (p) {
627: #ifdef SERVER
628: if ((rv = getartbyid(p)) != NULL) {
629: strcpy(fname, article_name());
630: (void) fclose(rv);
631: rv = NULL;
632: }
633: else
634: xerror("Cannot hfopen article %s", artid);
635: #else /* !SERVER */
636: (void) strcpy(fname, dirname(p));
637: #endif /* !SERVER */
638: rv = fopen(fname, "r"); /* NOT xfopen! */
639: if (rv == NULL)
640: xerror("Cannot hfopen article %s", artid);
641: }
642: #ifdef SERVER
643: (void) unlink(fname);
644: #endif /* !SERVER */
645: return rv;
646: }
647: #ifndef SERVER
648: # ifdef DBM
649: /*
650: ** Avoid problems of multiple dbminit calls.
651: */
652: initdbm(name)
653: char *name;
654: {
655: static int called = 0;
656:
657: if (called != 0)
658: return;
659: called = 1;
660: (void) dbminit(name);
661: }
662: # endif /* DBM */
663: #endif /* !SERVER */
664:
665: #ifndef BSD4_2
666: /*
667: * move n bytes from a to b
668: */
669: bcopy(a, b, n)
670: register char *a, *b;
671: register n;
672: {
673: while (--n >= 0)
674: *b++ = *a++;
675: }
676: #endif
677:
678: #if !defined(BSD4_2)
679: rename(from,to)
680: register char *from, *to;
681: {
682: (void) unlink(to);
683: if (link(from, to) < 0)
684: return -1;
685:
686: (void) unlink(from);
687: return 0;
688: }
689: #endif /* !BSD4_2 */
690:
691: #ifndef DBM
692: /*
693: ** Generate the appropriate history subfile name
694: */
695: char *
696: histfile(hline)
697: char *hline;
698: {
699: char chr; /* least significant digit of article number */
700: static char subfile[BUFLEN];
701:
702: chr = findhfdigit(hline);
703: sprintf(subfile, "%s.d/%c", ARTFILE, chr);
704: return subfile;
705: }
706:
707: findhfdigit(fn)
708: char *fn;
709: {
710: register char *p;
711: register int chr;
712:
713: p = index(fn, '@');
714: if (p != NULL && p > fn)
715: chr = *(p - 1);
716: else
717: chr = '0';
718: if (!isdigit(chr))
719: chr = '0';
720: return chr;
721: }
722: #endif /* !DBM */
723:
724: #ifdef VMS
725: /*
726: * These functions open an article with one level of indirection,
727: * to support symbolic links. xart_open exits if the open fails.
728: */
729: FILE *
730: xart_open (filename,mode)
731: char *filename,*mode;
732: {
733: FILE *fp = art_open (filename, mode);
734: extern int errno;
735: if (fp == NULL)
736: xerror("Cannot open article %s (%s): %s\n",
737: filename, mode, errmsg(errno));
738: return fp;
739: }
740:
741: FILE *
742: art_open (filename,mode)
743: char *filename,*mode;
744: {
745: char linkfile[BUFSIZ];
746: FILE *fp;
747:
748: if ((fp = fopen (filename, mode)) == NULL)
749: return NULL;
750: if (fgets (linkfile, BUFSIZ, fp) == NULL || linkfile[0] != '/') {
751: rewind (fp);
752: return fp;
753: }
754: /* Chase the symbolic link. */
755: (void) fclose (fp);
756: if ((fp = fopen (linkfile, mode)) == NULL)
757: /* Clean up dangling link, if we have the power. Ignore error if we don't. */
758: (void) unlink (filename);
759: return fp;
760: }
761: #endif /* VMS */
762:
763: /*
764: * Generate the name of the person responsible for posting this article,
765: * in order to check that two articles were posted by the same person.
766: */
767: char *
768: senderof(hp)
769: struct hbuf *hp;
770: {
771: register char *q, *tp;
772: char *tailpath();
773: static char senderbuf[BUFLEN];
774:
775: if (hp->sender[0])
776: tp = hp->sender;
777: else if (hp->from[0])
778: tp = hp->from;
779: else
780: tp = tailpath(hp);
781:
782: (void) strncpy(senderbuf, tp, BUFLEN);
783: /* Remove full name */
784: q = index(senderbuf, ' ');
785: if (q)
786: *q = '\0';
787:
788: return senderbuf;
789: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.