![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to header.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [CSRG BSD Unix] / 43BSD / contrib / news / src|
Return to header.c CVS log | Up to [CSRG BSD Unix] / 43BSD / contrib / news / src |
1.1 root 1: /*
2: * This software is Copyright (c) 1986 by Rick Adams.
3: *
4: * Permission is hereby granted to copy, reproduce, redistribute or
5: * otherwise use this software as long as: there is no monetary
6: * profit gained specifically from the use or reproduction or this
7: * software, it is not sold, rented, traded or otherwise marketed, and
8: * this copyright notice is included prominently in any copy
9: * made.
10: *
11: * The author make no claims as to the fitness or correctness of
12: * this software for any use whatsoever, and it is provided as is.
13: * Any use of this software is at the user's own risk.
14: *
15: * header.c - header functions plus some other goodies
16: */
17:
18: #ifdef SCCSID
19: static char *SccsId = "@(#)header.c 2.41 3/19/86";
20: #endif /* SCCSID */
21:
22: #include <stdio.h>
23: #include "params.h"
24:
25: char *hfgets();
26:
27: char *news_version = NEWS_VERSION;
28:
29: /*
30: * Read header from file fp into *hp. If wholething is FALSE,
31: * it's an incremental read, otherwise start from scratch.
32: * Return (FILE *) if header okay, else NULL.
33: */
34: FILE *
35: hread(hp, fp, wholething)
36: register struct hbuf *hp;
37: FILE *fp;
38: int wholething;
39: {
40: register int len;
41: register int i;
42: #ifdef OLD
43: char *p;
44: #endif /* OLD */
45:
46: if (wholething) {
47: for(i=0;i<NUNREC;i++)
48: if (hp->unrec[i] != NULL)
49: free(hp->unrec[i]);
50: else
51: break;
52: bzero((char *)hp, sizeof (*hp));
53: for (i=0;i<NUNREC;i++)
54: hp->unrec[i] = NULL;
55: }
56:
57: /* Check that it's a B news style header. */
58: if (hfgets(bfr, PATHLEN, fp) != NULL && isalpha(bfr[0])
59: && index(bfr, ':'))
60: if (frmread(fp, hp))
61: goto strip;
62:
63: if (!nstrip(bfr+1))
64: return NULL;
65:
66: /* It's not. Try A news (begins with PROTO). */
67: if (bfr[0] != PROTO)
68: return NULL;
69: #ifndef OLD
70: logerr("Can not process A news format article without OLD defined");
71: #else /* OLD */
72: /* Read in an A news format article. */
73: p = index(bfr+1, '.');
74: if (p == NULL) {
75: (void) strcpy(hp->ident, bfr+1);
76: return NULL;
77: }
78: *p++ = '\0';
79: (void) sprintf(hp->ident, "<%s@%s%s>", p, bfr+1, ".UUCP");
80:
81: /* Newsgroup List */
82: if (hfgets(hp->nbuf, BUFLEN, fp) == NULL || !nstrip(hp->nbuf))
83: return NULL;
84: /* source path */
85: if (hfgets(hp->path, PATHLEN, fp) == NULL || !nstrip(hp->path))
86: return NULL;
87: /* date */
88: if (hfgets(hp->subdate, DATELEN, fp) == NULL || !nstrip(hp->subdate))
89: return NULL;
90: /* title */
91: if (hfgets(hp->title, BUFLEN, fp) == NULL || !nstrip(hp->title))
92: return NULL;
93: #endif /* OLD */
94:
95: strip: /* strip off sys! from front of path. */
96: if (strncmp(FULLSYSNAME, hp->path, (len = strlen(FULLSYSNAME))) == 0
97: && index(NETCHRS, hp->path[len]))
98: (void) strcpy(hp->path, &(hp->path[len+1]));
99: lcase(hp->nbuf);
100:
101: /* Intuit the From: line if only a path was given. */
102: if (wholething) {
103: #ifdef OLD
104: if (hp->from[0] == '\0')
105: intuitfrom(hp);
106: else
107: #endif /* OLD */
108: fixfrom(hp);
109: }
110:
111: return fp;
112: }
113:
114:
115: /*
116: * Get header info from mail-format file.
117: * Return non-zero on success.
118: */
119: #include <ctype.h>
120: #define FROM 1
121: #define NEWSGROUP 2
122: #define TITLE 3
123: #define SUBMIT 4
124: #define RECEIVE 5
125: #define EXPIRE 6
126: #define ARTICLEID 7
127: #define MESSAGEID 8
128: #define REPLYTO 9
129: #define FOLLOWID 10
130: #define CONTROL 11
131: #define SENDER 12
132: #define FOLLOWTO 13
133: #define PATH 14
134: #define POSTVERSION 15
135: #define RELAYVERSION 16
136: #define DISTRIBUTION 17
137: #define ORGANIZATION 18
138: #define NUMLINES 19
139: #define KEYWORDS 20
140: #define APPROVED 21
141: #define NFID 22
142: #define NFFROM 23
143: #define XREF 24
144: #define SUMMARY 25
145: #define XPATH 26
146: #define OTHER 99
147:
148: char *malloc();
149:
150: frmread(fp, hp)
151: register FILE *fp;
152: register struct hbuf *hp;
153: {
154: int unreccnt = 0;
155: register int i;
156: long curpos;
157:
158: i = type(bfr);
159: do {
160: curpos = ftell(fp);
161: switch (i) {
162: case PATH:
163: getfield(hp->path, sizeof(hp->path));
164: break;
165: case FROM:
166: getfield(hp->from, sizeof(hp->from));
167: break;
168: case NEWSGROUP:
169: getfield(hp->nbuf, sizeof(hp->nbuf));
170: break;
171: case TITLE:
172: getfield(hp->title, sizeof(hp->title));
173: break;
174: case SUBMIT:
175: getfield(hp->subdate, sizeof(hp->subdate));
176: break;
177: case EXPIRE:
178: getfield(hp->expdate, sizeof(hp->expdate));
179: break;
180: #ifdef OLD
181: case ARTICLEID:
182: /* Believe Message-ID in preference to Article-ID */
183: if (hp->ident[0] == '\0') {
184: register char *p;
185: char msgb[NAMELEN];
186: getfield(msgb, sizeof msgb);
187: p = index(msgb, '.');
188: if (p == NULL) {
189: (void) strcpy(hp->ident, msgb);
190: } else {
191: *p++ = '\0';
192: (void) sprintf(hp->ident, "<%s@%s%s>", p, msgb, ".UUCP");
193: }
194: }
195: break;
196: #endif /* OLD */
197: case MESSAGEID:
198: getfield(hp->ident, sizeof(hp->ident));
199: break;
200: case REPLYTO:
201: getfield(hp->replyto, sizeof(hp->replyto));
202: break;
203: case FOLLOWID:
204: getfield(hp->followid, sizeof(hp->followid));
205: break;
206: case SENDER:
207: getfield(hp->sender, sizeof(hp->sender));
208: break;
209: case FOLLOWTO:
210: getfield(hp->followto, sizeof(hp->followto));
211: break;
212: case CONTROL:
213: getfield(hp->ctlmsg, sizeof(hp->ctlmsg));
214: break;
215: case DISTRIBUTION:
216: getfield(hp->distribution, sizeof(hp->distribution));
217: break;
218: case ORGANIZATION:
219: getfield(hp->organization, sizeof(hp->organization));
220: break;
221: case NUMLINES:
222: getfield(hp->numlines, sizeof(hp->numlines));
223: hp->intnumlines = atoi(hp->numlines);
224: break;
225: case KEYWORDS:
226: getfield(hp->keywords, sizeof(hp->keywords));
227: break;
228: case APPROVED:
229: getfield(hp->approved, sizeof(hp->approved));
230: break;
231: case NFID:
232: getfield(hp->nf_id, sizeof(hp->nf_id));
233: break;
234: case NFFROM:
235: getfield(hp->nf_from, sizeof(hp->nf_from));
236: break;
237: /* discard these lines */
238: case XREF:
239: case XPATH:
240: case RELAYVERSION:
241: case POSTVERSION:
242: case RECEIVE:
243: break;
244: case SUMMARY:
245: getfield(hp->summary, sizeof(hp->summary));
246: break;
247: case OTHER:
248: if (unreccnt < NUNREC) {
249: if ((hp->unrec[unreccnt] = malloc((unsigned)(strlen(bfr) + 1))) != NULL ) {
250: (void) strcpy(hp->unrec[unreccnt], bfr);
251: (void) nstrip(hp->unrec[unreccnt]);
252: unreccnt++;
253: } else
254: xerror("frmread: out of memory");
255: }
256: break;
257: }
258: } while ((i = type(hfgets(bfr, LBUFLEN, fp))) > 0);
259:
260: if (*bfr != '\n')
261: fseek(fp, curpos, 0);
262: if ((hp->from[0] || hp->path[0]) && hp->subdate[0] && hp->ident[0])
263: return TRUE;
264: return FALSE;
265: }
266:
267: #ifdef OLD
268: /*
269: * There was no From: line in the message (because it was generated by
270: * an old news program). Guess what it should have been and create it.
271: */
272: intuitfrom(hp)
273: register struct hbuf *hp;
274: {
275: char *tp;
276: char *user, *host;
277: char *tailpath(), *rindex();
278: char *at, *dot;
279: char pathbuf[PATHLEN];
280: char fullname[BUFLEN];
281:
282: tp = tailpath(hp);
283: user = rindex(tp, '!');
284: if (user == NULL)
285: user = tp;
286: else
287: *user++ = '\0';
288:
289: /* Check for an existing Internet address on the end. */
290: at = index(user, '@');
291: if (at) {
292: dot = index(at, '.');
293: if (dot) {
294: (void) strcpy(hp->from, user);
295: return;
296: }
297: /* @ signs are illegal except for the biggie, so */
298: *at = '%';
299: }
300:
301: if (tp[0] == '.')
302: host = index(tp, '!') + 1;
303: else if (user == tp)
304: host = FULLSYSNAME;
305: else
306: host = tp;
307:
308: tp = index(host, '@');
309: if (tp != NULL)
310: *tp = 0;
311: (void) sprintf(hp->from, "%s@%s%s", user, host, MYDOMAIN);
312:
313: skin(pathbuf, fullname, hp->path); /* remove RFC822-style comments */
314: if (fullname[0] != '\0') {
315: strcat(hp->from, " (");
316: (void) strcat(hp->from, fullname);
317: strcat(hp->from, ")");
318: }
319: strcpy(hp->path, pathbuf); /* and stick it back in */
320: }
321: #endif /* OLD */
322:
323: /*
324: * Canonicalize the "From:" line into the form
325: *
326: * From: <mail-address> (full-name)
327: *
328: * RFC822 doesn't require the comment to be at the end of the string
329: * like that.
330: */
331: fixfrom(hp)
332: register struct hbuf *hp;
333: {
334: char frombuf[PATHLEN];
335: char fullname[BUFLEN];
336:
337: skin(frombuf, fullname, hp->from); /* remove RFC822-style comments */
338: if (fullname[0] != '\0') {
339: strcat(frombuf, " (");
340: strcat(frombuf, fullname);
341: strcat(frombuf, ")");
342: }
343: strcpy(hp->from, frombuf); /* stick the canonicalized "from" back in */
344: }
345:
346: skin(name, fullname, hfield)
347: char *name;
348: char *fullname;
349: char *hfield;
350: {
351: register int c;
352: register char *cp, *cp2;
353: char *bufend;
354: int gotlt, parenlev, lastsp;
355: int seenfullname = FALSE;
356:
357: *fullname = '\0'; /* no full name yet */
358: if (strpbrk(hfield, "(< ") == NULL) { /* include ',' ?? */
359: strcpy(name, hfield);
360: return;
361: }
362: gotlt = 0;
363: parenlev = 0;
364: lastsp = 0;
365: bufend = name;
366: for (cp = hfield, cp2 = bufend; c = *cp++; ) {
367: switch (c) {
368: case '(':
369: /*
370: * Start of a "comment".
371: * Ignore it, or save it in "fullname" if we haven't
372: * seen a comment yet.
373: */
374: parenlev++;
375: while ((c = *cp) != 0) {
376: cp++;
377: switch (c) {
378: case '\\':
379: if ((c = *cp) == 0)
380: goto outcm;
381: cp++;
382: break;
383: case '(':
384: parenlev++;
385: break;
386: case ')':
387: parenlev--;
388: if (parenlev == 0)
389: goto outcm;
390: break;
391: }
392: if (!seenfullname)
393: *fullname++ = c;
394: }
395: outcm:
396: parenlev = 0;
397: lastsp = 0;
398: if (!seenfullname) {
399: *fullname = '\0';
400: seenfullname = TRUE; /* only extract first comment */
401: }
402: break;
403:
404: case '"':
405: /*
406: * Start of a "quoted-string".
407: * Copy it in its entirety.
408: */
409: while ((c = *cp) != 0) {
410: cp++;
411: switch (c) {
412: case '\\':
413: if ((c = *cp) == 0)
414: goto outqs;
415: cp++;
416: break;
417: case '"':
418: goto outqs;
419: }
420: *cp2++ = c;
421: }
422: outqs:
423: lastsp = 0;
424: break;
425:
426: case ' ':
427: if (cp[0] == 'a' && cp[1] == 't' && cp[2] == ' ')
428: cp += 3, *cp2++ = '@';
429: else
430: if (cp[0] == '@' && cp[1] == ' ')
431: cp += 2, *cp2++ = '@';
432: else
433: lastsp = 1;
434: break;
435:
436: case '<':
437: if (!seenfullname) {
438: *cp2 = '\0';
439: strcpy(fullname, name);
440: seenfullname = TRUE;
441: }
442: cp2 = bufend;
443: gotlt++;
444: lastsp = 0;
445: break;
446:
447: case '>':
448: if (gotlt) {
449: gotlt = 0;
450: /*
451: * this doesn't seem reasonable, what about (,)
452: * or "," ??
453: */
454: while (*cp != ',' && *cp != 0)
455: cp++;
456: if (*cp == 0 )
457: goto done;
458: *cp2++ = ',';
459: *cp2++ = ' ';
460: bufend = cp2;
461: break;
462: }
463:
464: /* Fall into . . . */
465:
466: default:
467: if (lastsp) {
468: lastsp = 0;
469: *cp2++ = ' ';
470: }
471: *cp2++ = c;
472: break;
473: }
474: }
475: done:
476: *cp2 = 0;
477: }
478:
479:
480: #ifdef OLD
481: char *
482: oident(ident)
483: char *ident;
484: {
485: char lbuf[BUFLEN];
486: static char oidbuf[BUFLEN];
487: register char *p, *q;
488:
489: (void) strcpy(lbuf, ident);
490: p = index(lbuf, '@');
491: if (p == NULL)
492: return ident;
493: *p++ = '\0';
494: q = index(p, '.');
495: if (q == NULL)
496: q = index(p, '>');
497: if (q)
498: *q++ = '\0';
499: p[SNLN] = '\0';
500: (void) sprintf(oidbuf, "%s.%s", p, lbuf+1);
501: return oidbuf;
502: }
503: #endif /* OLD */
504:
505: /*
506: * Get the given field of a header (char * parm) from bfr, but only
507: * if there's something actually there (after the colon). Don't
508: * bother if we already have an entry for this field.
509: */
510: getfield(hpfield, size)
511: char *hpfield;
512: int size;
513: {
514: register char *ptr;
515:
516: if (hpfield[0])
517: return;
518: for (ptr = index(bfr, ':'); isspace(*++ptr); )
519: ;
520: if (*ptr != '\0') {
521: (void) strncpy(hpfield, ptr, size - 1);
522: (void) nstrip(hpfield);
523: }
524: }
525:
526:
527: #define its(type) (prefix(ptr, type))
528: type(ptr)
529: register char *ptr;
530: {
531: register char *colon, *space;
532:
533: if (ptr == NULL)
534: return FALSE;
535: if (!isalpha(*ptr))
536: return FALSE;
537: colon = index(ptr, ':');
538: space = index(ptr, ' ');
539: if (!colon || space && space < colon)
540: return FALSE;
541: if (its("From: "))
542: if (index(ptr, '@') || !index(ptr, '!'))
543: return FROM;
544: else
545: return PATH;
546: if (its("Path: "))
547: return PATH;
548: if (its("Newsgroups: "))
549: return NEWSGROUP;
550: if (its("Subject: "))
551: return TITLE;
552: if (its("Date: "))
553: return SUBMIT;
554: if (its("Date-Received: "))
555: return RECEIVE;
556: #ifdef OLD
557: if (its("Title: "))
558: return TITLE;
559: if (its("Posted: "))
560: return SUBMIT;
561: if (its("Received: "))
562: return RECEIVE;
563: #endif /* OLD */
564: if (its("Expires: "))
565: return EXPIRE;
566: if (its("Article-I.D.: "))
567: return ARTICLEID;
568: if (its("Message-ID: "))
569: return MESSAGEID;
570: if (its("Reply-To: "))
571: return REPLYTO;
572: if (its("References: "))
573: return FOLLOWID;
574: if (its("Control: "))
575: return CONTROL;
576: if (its("Sender: "))
577: return SENDER;
578: if (its("Followup-To: "))
579: return FOLLOWTO;
580: if (its("Posting-Version: "))
581: return POSTVERSION;
582: if (its("Relay-Version: "))
583: return RELAYVERSION;
584: if (its("Distribution: "))
585: return DISTRIBUTION;
586: if (its("Organization: "))
587: return ORGANIZATION;
588: if (its("Lines: "))
589: return NUMLINES;
590: if (its("Summary: "))
591: return SUMMARY;
592: if (its("Keywords: "))
593: return KEYWORDS;
594: if (its("Approved: "))
595: return APPROVED;
596: if (its("Nf-ID: "))
597: return NFID;
598: if (its("Nf-From: "))
599: return NFFROM;
600: if (its("Xref: "))
601: return XREF;
602: if (its("Xpath: "))
603: return XPATH;
604: return OTHER;
605: }
606:
607: /*
608: * Generate the current version of the news software.
609: */
610:
611: static char *my_version = NULL;
612:
613: char *
614: genversion()
615: {
616: char rb[BUFLEN];
617: register char *t;
618:
619: if (my_version == NULL) {
620: #ifdef HIDDENNET
621: if (strcmp(LOCALSYSNAME, FULLSYSNAME))
622: (void) sprintf(rb, "version %s; site %s.%s%s",
623: news_version, LOCALSYSNAME, FULLSYSNAME, MYDOMAIN);
624: else
625: #endif /* !HIDDENNET */
626: (void) sprintf(rb, "version %s; site %s%s",
627: news_version, FULLSYSNAME, MYDOMAIN);
628: while (t = index(rb, '\t'))
629: *t = ' ';
630: my_version = malloc((unsigned)strlen(rb) + 1);
631: if (my_version == NULL)
632: xerror("malloc failed for my_version");
633: (void) strcpy(my_version, rb);
634: }
635: return my_version;
636: }
637:
638: /*
639: * Write header at 'hp' on stream 'fp' in B+ format. Include received date
640: * if wr is 1. Leave off sysname if wr is 2.
641: */
642: ihwrite(hp, fp, wr)
643: register struct hbuf *hp;
644: register FILE *fp;
645: int wr;
646: {
647: int iu;
648: time_t t;
649: time_t cgtdate();
650:
651: /*
652: * We're being tricky with Path/From because of upward compatibility
653: * issues. The new version considers From and Path to be separate.
654: * The old one thinks they both mean "Path" but only believes the
655: * first one it sees, so will ignore the second.
656: */
657: if (prefix(hp->path, FULLSYSNAME) &&
658: index(NETCHRS, hp->path[strlen(FULLSYSNAME)]))
659: fprintf(fp, "Path: %s\n", hp->path);
660: else
661: fprintf(fp, "Path: %s!%s\n", FULLSYSNAME, hp->path);
662: if (hp->from[0])
663: fprintf(fp, "From: %s\n", hp->from);
664:
665: fprintf(fp, "Newsgroups: %s\n", hp->nbuf);
666: fprintf(fp, "Subject: %s\n", hp->title);
667: fprintf(fp, "Message-ID: %s\n", hp->ident);
668: t = cgtdate(hp->subdate);
669: fprintf(fp, "Date: %s\n", arpadate(&t));
670: #ifdef OLD
671: fprintf(fp, "Article-I.D.: %s\n", oident(hp->ident));
672: fprintf(fp, "Posted: %s", ctime(&t));
673: #endif /* OLD */
674: if (*hp->expdate)
675: fprintf(fp, "Expires: %s\n", hp->expdate);
676: if (*hp->followid) {
677: register char *dp, *cp;
678:
679: dp = cp = hp->followid;
680: while (*cp != '\0')
681: if (*cp == '<' && *(cp + 1) == '>')
682: cp += 2;
683: else
684: *dp++ = *cp++;
685: *dp = '\0';
686: if (*hp->followid)
687: fprintf(fp, "References: %s\n", hp->followid);
688: }
689: if (*hp->ctlmsg)
690: fprintf(fp, "Control: %s\n", hp->ctlmsg);
691: if (*hp->sender)
692: fprintf(fp, "Sender: %s\n", hp->sender);
693: if (*hp->replyto)
694: fprintf(fp, "Reply-To: %s\n", hp->replyto);
695: if (*hp->followto)
696: fprintf(fp, "Followup-To: %s\n", hp->followto);
697: if (*hp->distribution)
698: fprintf(fp, "Distribution: %s\n", hp->distribution);
699: if (*hp->organization)
700: fprintf(fp, "Organization: %s\n", hp->organization);
701: if (*hp->numlines)
702: fprintf(fp, "Lines: %s\n", hp->numlines);
703: if (*hp->keywords)
704: fprintf(fp, "Keywords: %s\n", hp->keywords);
705: if (*hp->summary)
706: fprintf(fp, "Summary: %s\n", hp->summary);
707: if (*hp->approved)
708: fprintf(fp, "Approved: %s\n", hp->approved);
709: if (*hp->nf_id)
710: fprintf(fp, "Nf-ID: %s\n", hp->nf_id);
711: if (*hp->nf_from)
712: fprintf(fp, "Nf-From: %s\n", hp->nf_from);
713: #ifdef DOXREFS
714: if ( wr ==1 && *hp->xref)
715: fprintf(fp, "Xref: %s\n", hp->xref);
716: #endif /* DOXREFS */
717: for (iu = 0; iu < NUNREC; iu++) {
718: if (hp->unrec[iu])
719: fprintf(fp, "%s\n", &hp->unrec[iu][0]);
720: }
721: putc('\n', fp);
722: }
723:
724:
725: #ifndef BSD4_2
726: /*
727: * Set nc bytes, starting at cp, to zero.
728: */
729: bzero(cp, nc)
730: register char *cp;
731: register int nc;
732: {
733: if (nc > 0)
734: do {
735: *cp++ = 0;
736: } while (--nc);
737: }
738: #endif /* !BSD4_2 */
739:
740: /*
741: * hfgets is like fgets, but deals with continuation lines.
742: * It also ensures that even if a line that is too long is
743: * received, the remainder of the line is thrown away
744: * instead of treated like a second line.
745: */
746: char *
747: hfgets(buf, len, fp)
748: char *buf;
749: int len;
750: FILE *fp;
751: {
752: register int c;
753: register int n = 0;
754: register char *cp;
755:
756: cp = buf;
757: while (n < len && (c = getc(fp)) != EOF) {
758: if (c == '\n')
759: break;
760: if (isprint(c) || c == '\b' || c == ' ' || c == '\t') {
761: *cp++ = c;
762: n++;
763: }
764: }
765: if (c == EOF && cp == buf)
766: return NULL;
767: *cp = '\0';
768:
769: if (c != '\n') {
770: /* Line too long - part read didn't fit into a newline */
771: while ((c = getc(fp)) != '\n' && c != EOF)
772: ;
773: } else if (cp == buf) {
774: /* Don't look for continuation of blank lines */
775: *cp++ = '\n';
776: *cp = '\0';
777: return buf;
778: }
779:
780: while ((c = getc(fp)) == ' ' || c == '\t') { /* for each cont line */
781: /* Continuation line. */
782: if ((n += 2) < len) {
783: *cp++ = '\n';
784: *cp++ = c;
785: }
786: while ((c = getc(fp)) != '\n' && c != EOF)
787: if ((isprint(c) || c == '\b' || c == ' ' || c == '\t')
788: && n++ < len)
789: *cp++ = c;
790: }
791: if (n >= len - 1)
792: cp = buf + len - 2;
793: *cp++ = '\n';
794: *cp = '\0';
795: if (c != EOF)
796: (void) ungetc(c, fp); /* push back first char of next header */
797: return buf;
798: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.