|
|
1.1 root 1: #include <stdio.h>
2: #include <sys/param.h>
3: #include <sys/dir.h>
4: #include <sys/proc.h>
5: #include <sys/pte.h>
6: #include <a.out.h>
7: #include <sys/map.h>
8: #include <sys/user.h>
9: #include <sys/text.h>
10: #include <sys/cmap.h>
11: #include <sys/vm.h>
12:
13: /*
14: * Analyze - analyze a core (and optional paging area) saved from
15: * a virtual Unix system crash.
16: */
17:
18: int Dflg;
19: int dflg;
20: int vflg;
21: int mflg;
22: int fflg;
23: int sflg;
24:
25: /* use vprintf with care; it plays havoc with ``else's'' */
26: #define vprintf if (vflg) printf
27:
28: #define clear(x) ((int)x & 0x7fffffff)
29:
30: struct proc proc[NPROC];
31: struct text text[NTEXT];
32: struct map swapmap[SMAPSIZ];
33: struct cmap *cmap;
34: struct pte *usrpt;
35: struct pte *Usrptma;
36: int firstfree;
37: int maxfree;
38: int freemem;
39: struct pte p0br[ctopt(MAXTSIZ+MAXDSIZ+MAXSSIZ)][NPTEPG];
40: int pid;
41:
42: struct paginfo {
43: char z_type;
44: char z_count;
45: short z_pid;
46: struct pte z_pte;
47: } *paginfo;
48: #define ZLOST 0
49: #define ZDATA 1
50: #define ZSTACK 2
51: #define ZUDOT 3
52: #define ZPAGET 4
53: #define ZTEXT 5
54: #define ZFREE 6
55: #define ZINTRAN 7
56:
57: #define NDBLKS (2*SMAPSIZ)
58: struct dblks {
59: short d_first;
60: short d_size;
61: char d_type;
62: char d_index;
63: } dblks[NDBLKS];
64: int ndblks;
65:
66: #define DFREE 0
67: #define DDATA 1
68: #define DSTACK 2
69: #define DTEXT 3
70: #define DUDOT 4
71: #define DPAGET 5
72:
73: union {
74: char buf[UPAGES][512];
75: struct user U;
76: } u_area;
77: #define u u_area.U
78:
79: int fcore = -1;
80: int fswap = -1;
81:
82: struct nlist nl[] = {
83: #define X_PROC 0
84: "_proc", 0, 0, 0, 0,
85: #define X_USRPT 1
86: "_usrpt", 0, 0, 0, 0,
87: #define X_PTMA 2
88: "_Usrptma", 0, 0, 0, 0,
89: #define X_FIRSTFREE 3
90: "_firstfr", 0, 0, 0, 0,
91: #define X_MAXFREE 4
92: "_maxfree", 0, 0, 0, 0,
93: #define X_TEXT 5
94: "_text", 0, 0, 0, 0,
95: #define X_FREEMEM 6
96: "_freemem", 0, 0, 0, 0,
97: #define X_CMAP 7
98: "_cmap", 0, 0, 0, 0,
99: #define X_ECMAP 8
100: "_ecmap", 0, 0, 0, 0,
101: #define X_SWAPMAP 9
102: "_swapmap", 0, 0, 0, 0,
103: 0, 0, 0, 0, 0,
104:
105: };
106:
107: main(argc, argv)
108: int argc;
109: char **argv;
110: {
111: register struct nlist *np;
112: register struct proc *p;
113: register struct text *xp;
114: register struct pte *pte;
115: register int i;
116: int w, a;
117:
118: argc--, argv++;
119: while (argc > 0 && argv[0][0] == '-') {
120: register char *cp = *argv++;
121: argc--;
122: while (*++cp) switch (*cp) {
123:
124: case 'm':
125: mflg++;
126: break;
127:
128: case 'v':
129: vflg++;
130: break;
131:
132: case 's':
133: if (argc < 2)
134: goto usage;
135: if ((fswap = open(argv[0], 0)) < 0) {
136: perror(argv[0]);
137: exit(1);
138: }
139: argc--,argv++;
140: sflg++;
141: break;
142:
143: case 'f':
144: fflg++;
145: break;
146:
147: case 'D':
148: Dflg++;
149: break;
150:
151: case 'd':
152: dflg++;
153: break;
154:
155: default:
156: goto usage;
157: }
158: }
159: if (argc < 1) {
160: usage:
161: fprintf(stderr, "usage: analyze [ -vmfd ] [ -s swapfile ] corefile [ system ]\n");
162: exit(1);
163: }
164: close(0);
165: if ((fcore = open(argv[0], 0)) < 0) {
166: perror(argv[0]);
167: exit(1);
168: }
169: nlist(argc > 1 ? argv[1] : "/vmunix", nl);
170: if (nl[0].n_value == 0) {
171: fprintf(stderr, "%s: bad namelist\n",
172: argc > 1 ? argv[1] : "/vmunix");
173: exit(1);
174: }
175: for (np = nl; np->n_name[0]; np++)
176: vprintf("%8.8s %x\n", np->n_name ,np->n_value );
177: usrpt = (struct pte *)clear(nl[X_USRPT].n_value);
178: Usrptma = (struct pte *)clear(nl[X_PTMA].n_value);
179: firstfree = get(nl[X_FIRSTFREE].n_value);
180: maxfree = get(nl[X_MAXFREE].n_value);
181: freemem = get(nl[X_FREEMEM].n_value);
182: paginfo = (struct paginfo *)calloc(maxfree, sizeof (struct paginfo));
183: if (paginfo == NULL) {
184: fprintf(stderr, "maxfree %x?... out of mem!\n", maxfree);
185: exit(1);
186: }
187: vprintf("usrpt %x\nUsrptma %x\nfirstfree %x\nmaxfree %x\nfreemem %x\n",
188: usrpt, Usrptma, firstfree, maxfree, freemem);
189: lseek(fcore, (long)clear(nl[X_PROC].n_value), 0);
190: if (read(fcore, (char *)proc, sizeof proc) != sizeof proc) {
191: perror("proc read");
192: exit(1);
193: }
194: lseek(fcore, (long)clear(nl[X_TEXT].n_value), 0);
195: if (read(fcore, (char *)text, sizeof text) != sizeof text) {
196: perror("text read");
197: exit(1);
198: }
199: i = (get(nl[X_ECMAP].n_value) - get(nl[X_CMAP].n_value));
200: cmap = (struct cmap *)calloc(i, 1);
201: if (cmap == NULL) {
202: fprintf(stderr, "not enough mem for %x bytes of cmap\n", i);
203: exit(1);
204: }
205: lseek(fcore, (long)clear(get(nl[X_CMAP].n_value)), 0);
206: if (read(fcore, (char *)cmap, i) != i) {
207: perror("cmap read");
208: exit(1);
209: }
210: lseek(fcore, (long)clear(nl[X_SWAPMAP].n_value), 0);
211: if (read(fcore, (char *)swapmap, sizeof swapmap) != sizeof swapmap) {
212: perror("swapmap read");
213: exit(1);
214: }
215: for (p = &proc[1]; p < &proc[NPROC]; p++) {
216: p->p_p0br = (struct pte *)clear(p->p_p0br);
217: if (p->p_stat == 0)
218: continue;
219: printf("proc %d ", p->p_pid);
220: if (p->p_stat != SZOMB) {
221: if (getu(p))
222: continue;
223: u.u_procp = p;
224: }
225: if (p->p_stat == SZOMB) {
226: printf("zombie\n");
227: continue;
228: }
229: if (p->p_flag & SLOAD) {
230: printf("loaded, p0br %x, ", p->p_p0br);
231: p->p_szpt = u.u_pcb.pcb_szpt;
232: printf("%d pages of page tables:", p->p_szpt);
233: a = btokmx(p->p_p0br);
234: for (i = 0; i < p->p_szpt; i++) {
235: w = get(&Usrptma[a + i]);
236: printf(" %x", w & PG_PFNUM);
237: }
238: printf("\n");
239: for(i = 0; i < p->p_szpt; i++) {
240: w = get(&Usrptma[a + i]);
241: if (getpt(w, i))
242: count(p, (struct pte *)&w, ZPAGET);
243: }
244: } else {
245: /* i = ctopt(btoc(u.u_exdata.ux_dsize)); */
246: i = clrnd(ctopt(p->p_tsize + p->p_dsize + p->p_ssize));
247: printf("swapped, swaddr %x\n", p->p_swaddr);
248: duse(p->p_swaddr, clrnd(ctod(UPAGES)), DUDOT, p - proc);
249: duse(p->p_swaddr + ctod(UPAGES),
250: clrnd(i - p->p_tsize / NPTEPG), DPAGET, p - proc);
251: /* i, DPAGET, p - proc); */
252: }
253: p->p_p0br = (struct pte *)p0br;
254: p->p_textp = &text[p->p_textp - (struct text *)nl[X_TEXT].n_value];
255: if (p->p_pid == 2)
256: continue;
257: pdmap();
258: if ((p->p_flag & SLOAD) == 0)
259: continue;
260: pid = p->p_pid;
261: for (i = 0; i < p->p_tsize; i++) {
262: pte = tptopte(p, i);
263: if (pte->pg_fod || pte->pg_pfnum == 0)
264: continue;
265: if (pte->pg_pfnum >= firstfree && pte->pg_pfnum < maxfree && cmap[pgtocm(pte->pg_pfnum)].c_intrans)
266: count(p, pte, ZINTRAN);
267: else
268: count(p, pte, ZTEXT);
269: }
270: vprintf("\n");
271: for (i = 0; i < p->p_dsize; i++) {
272: pte = dptopte(p, i);
273: if (pte->pg_fod || pte->pg_pfnum == 0)
274: continue;
275: if (pte->pg_pfnum >= firstfree && pte->pg_pfnum < maxfree && cmap[pgtocm(pte->pg_pfnum)].c_intrans)
276: count(p, pte, ZINTRAN);
277: else
278: count(p, pte, ZDATA);
279: }
280: vprintf("\n");
281: for (i = 0; i < p->p_ssize; i++) {
282: pte = sptopte(p, i);
283: if (pte->pg_fod || pte->pg_pfnum == 0)
284: continue;
285: if (pte->pg_pfnum >= firstfree && pte->pg_pfnum < maxfree && cmap[pgtocm(pte->pg_pfnum)].c_intrans)
286: count(p, pte, ZINTRAN);
287: else
288: count(p, pte, ZSTACK);
289: }
290: vprintf("\n");
291: for (w = 0; w < UPAGES; w++) {
292: int l = p->p_addr[w];
293: count(p, (struct pte *)&l, ZUDOT);
294: }
295: vprintf("\n");
296: vprintf("\n");
297: }
298: for (xp = &text[0]; xp < &text[NTEXT]; xp++)
299: if (xp->x_iptr) {
300: duse(xp->x_daddr, xp->x_size, DTEXT, xp - text);
301: if (xp->x_flag & XPAGI)
302: duse(xp->x_daddr + xp->x_size,
303: clrnd(ctopt(xp->x_size)), DTEXT, xp - text);
304: }
305: dmcheck();
306: fixfree();
307: summary();
308: exit(0);
309: }
310:
311: pdmap()
312: {
313: register struct text *xp;
314:
315: if (fswap == -1 && (u.u_procp->p_flag & SLOAD) == 0)
316: return;
317: if (Dflg)
318: printf("disk for pid %d", u.u_procp->p_pid);
319: if (xp = u.u_procp->p_textp) {
320: xp = &text[xp - (struct text *)nl[X_TEXT].n_value];
321: if (Dflg)
322: printf(", text: %x<%x>", xp->x_daddr, xp->x_size);
323: }
324: pdmseg("data", &u.u_dmap, DDATA);
325: pdmseg("stack", &u.u_smap, DSTACK);
326: if (Dflg)
327: printf("\n");
328: }
329:
330: pdmseg(cp, dmp, type)
331: char *cp;
332: struct dmap *dmp;
333: {
334: register int i;
335: int b, rem;
336:
337: if (Dflg)
338: printf(", %s:", cp);
339: b = DMMIN;
340: for (i = 0, rem = dmp->dm_size; rem > 0; i++) {
341: if (Dflg)
342: printf(" %x<%x>", dmp->dm_map[i], rem < b ? rem : b);
343: duse(dmp->dm_map[i], b, type, u.u_procp - proc);
344: rem -= b;
345: if (b < DMMAX)
346: b *= 2;
347: }
348: }
349:
350: duse(first, size, type, index)
351: {
352: register struct dblks *dp;
353:
354: if (fswap == -1)
355: return;
356: dp = &dblks[ndblks];
357: if (++ndblks > NDBLKS) {
358: fprintf(stderr, "too many disk blocks, increase NDBLKS\n");
359: exit(1);
360: }
361: dp->d_first = first;
362: dp->d_size = size;
363: dp->d_type = type;
364: dp->d_index = index;
365: }
366:
367: dsort(d, e)
368: register struct dblks *d, *e;
369: {
370:
371: return (e->d_first - d->d_first);
372: }
373:
374: dmcheck()
375: {
376: register struct map *smp;
377: register struct dblks *d, *e;
378:
379: for (smp = swapmap; smp->m_size; smp++)
380: duse(smp->m_addr, smp->m_size, DFREE, 0);
381: qsort(dblks, ndblks, sizeof (struct dblks), dsort);
382: d = &dblks[ndblks - 1];
383: if (d->d_first > 1)
384: printf("lost swap map: start %x size %x\n", 1, d->d_first);
385: for (; d > dblks; d--) {
386: if (dflg)
387: dprint(d);
388: e = d - 1;
389: if (d->d_first + d->d_size > e->d_first) {
390: printf("overlap in swap mappings:\n");
391: dprint(d);
392: dprint(e);
393: } else if (d->d_first + d->d_size < e->d_first) {
394: printf("lost swap map: start %x size %x\n",
395: d->d_first + d->d_size,
396: e->d_first - (d->d_first + d->d_size));
397: }
398: }
399: if (dflg)
400: dprint(dblks);
401: if (sflg)
402: printf("swap space ends at %x\n", d->d_first + d->d_size);
403: }
404:
405: char *dnames[] = {
406: "DFREE",
407: "DDATA",
408: "DSTACK",
409: "DTEXT",
410: "DUDOT",
411: "DPAGET",
412: };
413:
414: dprint(d)
415: register struct dblks *d;
416: {
417:
418: printf("at %4x size %4x type %s", d->d_first, d->d_size,
419: dnames[d->d_type]);
420: switch (d->d_type) {
421:
422: case DSTACK:
423: case DDATA:
424: printf(" pid %d", proc[d->d_index].p_pid);
425: break;
426: }
427: printf("\n");
428: }
429:
430: getpt(x, i)
431: int x, i;
432: {
433:
434: lseek(fcore, (long)ctob((x & PG_PFNUM)), 0);
435: if (read(fcore, (char *)(p0br[i]), NBPG) != NBPG) {
436: perror("read");
437: fprintf(stderr, "getpt error reading frame %x\n", clear(x));
438: return (0);
439: }
440: return (1);
441: }
442:
443: checkpg(p, pte, type)
444: register struct pte *pte;
445: register struct proc *p;
446: int type;
447: {
448: char corepg[NBPG], swapg[NBPG];
449: register int i, count, dblock;
450: register int pfnum = pte->pg_pfnum;
451:
452: if (type == ZPAGET || type == ZUDOT)
453: return (0);
454: lseek(fcore, (long)(NBPG * pfnum), 0);
455: if (read(fcore, corepg, NBPG) != NBPG){
456: perror("read");
457: fprintf(stderr, "Error reading core page %x\n", pfnum);
458: return (0);
459: }
460: switch (type) {
461:
462: case ZDATA:
463: if (ptetodp(p, pte) >= u.u_dmap.dm_size)
464: return (0);
465: break;
466:
467: case ZTEXT:
468: break;
469:
470: case ZSTACK:
471: if (ptetosp(p, pte) >= u.u_smap.dm_size)
472: return (0);
473: break;
474:
475: default:
476: return(0);
477: break;
478: }
479: dblock = vtod(p, ptetov(p, pte), &u.u_dmap, &u.u_smap);
480: vprintf(" %x", dblock);
481: if (pte->pg_fod || pte->pg_pfnum == 0)
482: return (0);
483: if (cmap[pgtocm(pte->pg_pfnum)].c_intrans || pte->pg_m || pte->pg_swapm)
484: return (0);
485: lseek(fswap, (long)(NBPG * dblock), 0);
486: if (read(fswap, swapg, NBPG) != NBPG) {
487: fprintf(stderr,"swap page %x: ", dblock);
488: perror("read");
489: }
490: count = 0;
491: for (i = 0; i < NBPG; i++)
492: if (corepg[i] != swapg[i])
493: count++;
494: if (count == 0)
495: vprintf("\tsame");
496: return (count);
497: }
498:
499: getu(p)
500: register struct proc *p;
501: {
502: int i, w, errs = 0;
503:
504: for (i = 0; i < UPAGES; i++) {
505: if (p->p_flag & SLOAD) {
506: w = p->p_addr[i];
507: lseek(fcore, (long)(NBPG * clear(w)), 0);
508: if (read(fcore, u_area.buf[i], NBPG) != NBPG)
509: perror("core u. read"), errs++;
510: } else if (fswap >= 0) {
511: lseek(fswap, (long)(NBPG * (p->p_swaddr+i)), 0);
512: if (read(fswap, u_area.buf[i], NBPG) != NBPG)
513: perror("swap u. read"), errs++;
514: }
515: }
516: return (errs);
517: }
518:
519: char *typepg[] = {
520: "lost",
521: "data",
522: "stack",
523: "udot",
524: "paget",
525: "text",
526: "free",
527: "intransit",
528: };
529:
530: count(p, pte, type)
531: struct proc *p;
532: register struct pte *pte;
533: int type;
534: {
535: register int pfnum = pte->pg_pfnum;
536: register struct paginfo *zp = &paginfo[pfnum];
537: int ndif;
538: #define zprintf if (type==ZINTRAN || vflg) printf
539:
540: if (type == ZINTRAN && pfnum == 0)
541: return;
542: zprintf("page %x %s", pfnum, typepg[type]);
543: if (sflg == 0 || (ndif = checkpg(p, pte, type)) == 0) {
544: zprintf("\n");
545: } else {
546: if (vflg == 0 && type != ZINTRAN)
547: printf("page %x %s,", pfnum, typepg[type]);
548: printf(" %d bytes differ\n",ndif);
549: }
550: if (pfnum < firstfree || pfnum > maxfree) {
551: printf("page number out of range:\n");
552: printf("\tpage %x type %s pid %d\n", pfnum, typepg[type], pid);
553: return;
554: }
555: if (bad(zp, type)) {
556: printf("dup page pte %x", *(int *)pte);
557: dumpcm("", pte->pg_pfnum);
558: dump(zp);
559: printf("pte %x and as %s in pid %d\n", zp->z_pte, typepg[type], pid);
560: return;
561: }
562: zp->z_type = type;
563: zp->z_count++;
564: zp->z_pid = pid;
565: zp->z_pte = *pte;
566: }
567:
568: bad(zp, type)
569: struct paginfo *zp;
570: {
571: if (type == ZTEXT) {
572: if (zp->z_type != 0 && zp->z_type != ZTEXT)
573: return (1);
574: return (0);
575: }
576: return (zp->z_count);
577: }
578:
579: dump(zp)
580: struct paginfo *zp;
581: {
582:
583: printf("page %x type %s pid %d ", zp - paginfo, typepg[zp->z_type], zp->z_pid);
584: }
585:
586: summary()
587: {
588: register int i;
589: register struct paginfo *zp;
590: register int pfnum;
591:
592: for (i = firstfree + UPAGES; i < maxfree; i++) {
593: zp = &paginfo[i];
594: if (zp->z_type == ZLOST)
595: dumpcm("lost", i);
596: pfnum = pgtocm(i);
597: if ((cmap[pfnum].c_flag & MLOCK) && !(cmap[pfnum].c_flag & MSYS))
598: dumpcm("locked", i);
599: if (mflg)
600: dumpcm("mem", i);
601: }
602: }
603:
604: dumpcm(cp, pg)
605: char *cp;
606: int pg;
607: {
608: int pslot;
609: int cm;
610:
611: printf("%s page %x ", cp, pg);
612: cm = pgtocm(pg);
613: printf("\t[%x, %x", cmap[cm].c_page, cmap[cm].c_ndx);
614: if ((cmap[cm].c_flag&MTEXT) == 0)
615: printf(" (=pid %d)", proc[cmap[cm].c_ndx].p_pid);
616: else {
617: pslot=(text[cmap[cm].c_ndx].x_caddr - (struct proc *)nl[X_PROC].n_value);
618: printf(" (=pid");
619: for(;;) {
620: printf(" %d", proc[pslot].p_pid);
621: if (proc[pslot].p_xlink == 0)
622: break;
623: pslot=(proc[pslot].p_xlink - (struct proc *)nl[X_PROC].n_value);
624: }
625: printf(")");
626: }
627:
628: #define Mflag(x,y) if (cmap[cm].c_flag&x) printf(y);
629: Mflag(MTEXT, " MTEXT");
630: Mflag(MDATA, " MDATA");
631: Mflag(MSTACK, "MSTACK");
632: Mflag(MSYS, " MSYS");
633: Mflag(MFREE, " MFREE");
634: Mflag(MLOCK, " MLOCK");
635: printf("]\n");
636: }
637:
638: fixfree()
639: {
640: register int i, next, prev;
641:
642: next = CMHEAD;
643: for (i=freemem/CLSIZE; --i >=0; ) {
644: prev = next;
645: next = cmap[next].c_next;
646: if ((cmap[next].c_flag&MFREE) == 0) {
647: printf("link to non free block: in %x to %x\n", cmtopg(prev), cmtopg(next));
648: dumpcm("bad free link in", cmtopg(prev));
649: dumpcm("to non free block", cmtopg(next));
650: }
651: if (cmtopg(next) > maxfree) {
652: printf("free list link out of range: in %x to %x\n", cmtopg(prev), cmtopg(next));
653: dumpcm("bad link in", cmtopg(prev));
654: }
655: paginfo[cmtopg(next)].z_type = ZFREE;
656: if (fflg)
657: dumpcm("free", cmtopg(next));
658: paginfo[cmtopg(next)+1].z_type = ZFREE;
659: if (fflg)
660: dumpcm("free", cmtopg(next)+1);
661: }
662: }
663:
664: get(loc)
665: unsigned loc;
666: {
667: int x;
668:
669: lseek(fcore, (long)clear(loc), 0);
670: if (read(fcore, (char *)&x, sizeof (int)) != sizeof (int)) {
671: perror("read");
672: fprintf(stderr, "get failed on %x\n", clear(loc));
673: return (0);
674: }
675: return (x);
676: }
677: /*
678: * Convert a virtual page number
679: * to its corresponding disk block number.
680: * Used in pagein/pageout to initiate single page transfers.
681: */
682: vtod(p, v, dmap, smap)
683: register struct proc *p;
684: register struct dmap *dmap, *smap;
685: {
686: struct dblock db;
687:
688: if (v < p->p_tsize)
689: return(p->p_textp->x_daddr + v);
690: if (isassv(p, v))
691: vstodb(vtosp(p, v), 1, smap, &db, 1);
692: else
693: vstodb(vtodp(p, v), 1, dmap, &db, 0);
694: return (db.db_base);
695: }
696:
697: /*
698: * Convert a pte pointer to
699: * a virtual page number.
700: */
701: ptetov(p, pte)
702: register struct proc *p;
703: register struct pte *pte;
704: {
705:
706: if (isatpte(p, pte))
707: return (tptov(p, ptetotp(p, pte)));
708: else if (isadpte(p, pte))
709: return (dptov(p, ptetodp(p, pte)));
710: else
711: return (sptov(p, ptetosp(p, pte)));
712: }
713:
714: /*
715: * Given a base/size pair in virtual swap area,
716: * return a physical base/size pair which is the
717: * (largest) initial, physically contiguous block.
718: */
719: vstodb(vsbase, vssize, dmp, dbp, rev)
720: register int vsbase;
721: int vssize;
722: register struct dmap *dmp;
723: register struct dblock *dbp;
724: {
725: register int blk = DMMIN;
726: register swblk_t *ip = dmp->dm_map;
727:
728: if (vsbase < 0 || vsbase + vssize > dmp->dm_size)
729: panic("vstodb");
730: while (vsbase >= blk) {
731: vsbase -= blk;
732: if (blk < DMMAX)
733: blk *= 2;
734: ip++;
735: }
736: dbp->db_size = min(vssize, blk - vsbase);
737: dbp->db_base = *ip + (rev ? blk - (vsbase + vssize) : vsbase);
738: }
739:
740: panic(cp)
741: char *cp;
742: {
743: printf("panic!: %s\n", cp);
744: }
745:
746: min(a, b)
747: {
748: return (a < b ? a : b);
749: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.