![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to sdi.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Research Unix] / researchv9 / cmd / sun / as|
Return to sdi.c CVS log | Up to [Research Unix] / researchv9 / cmd / sun / as |
1.1 root 1: #ifndef lint
2: static char sccsid[] = "@(#)sdi.c 1.1 86/02/03 Copyr 1985 Sun Micro";
3: #endif
4:
5: /*
6: * Copyright (c) 1985 by Sun Microsystems, Inc.
7: */
8:
9: #include "as.h"
10:
11: /*
12: * module to handle span-dependent instructions, e.g. jbr
13: * see CACM Vol 21 No 4 April 1978 for a description of the
14: * algorithm for resolving sdi's between the 1st and 2nd pass
15: */
16:
17: #define SQUID( x ) ( ((x)==C_TEXT)||(rflag && ((x) != C_BSS)) )
18: #define INFINITY 0x7fffffff /* larger than any address */
19:
20: struct sdi { /* information for span dependent instructions (sdi's) */
21: struct sdi *sdi_next; /* next sdi in list */
22: long int sdi_loc; /* location of the sdi */
23: union{
24: struct sym_bkt *sdi_Sym;/* symbol part of the sdi address */
25: int *sdi_Add;
26: }su;
27: #define sdi_sym su.sdi_Sym
28: #define sdi_address su.sdi_Add
29: long int sdi_off; /* offset part of the sdi address */
30: short sdi_leng; /* actual length of the sdi */
31: long int sdi_base; /* origin of the span of the sdi */
32: short sdi_flags; /* bits: is/was it a ddi? */
33: # if EBUG
34: unsigned sdi_lineno;
35: # endif EBUG
36: } *sdi_list; /* linked list of descriptors */
37:
38: struct sdi *ddi_list;
39:
40: /* sdi_flag bit */
41: #define SDI_TYPE 7 /* SDI4, SDI6, SDI8, SDIP */
42: #define SDI_DDI 010
43: #define SDI_EXTERN 020
44:
45: /* SDI4, SDI6, SDI8, SDIP, SDIX, SDIL */
46: static int glen[] = { 4, 6, 8, 4, 6, 4 };
47:
48:
49: /* grab a new sdi descriptor off of the free list; allocate space
50: * for a new free list if necessary
51: * stolen from the symbol-table manager
52: * We keep the sdi and ddi lists separate because:
53: * 1) we want to keep locality in the sdi list
54: * 2) we want to give back the ddi list
55: */
56:
57: #define SEGALLOC 100
58: static
59: struct seglist { int allocsize, nalloced; struct sdi *nextfree, *head; char *segname;}
60: sdilist = { SEGALLOC, 0, NULL, NULL, "sdi" },
61: ddilist = { SEGALLOC, 0, NULL, NULL, "ddi" };
62:
63: static long total_delta = 0;
64:
65: struct sdi *
66: sdi_alloc(s)
67: register struct seglist *s;
68: {
69: register struct sdi *sdp, *sdi_free;
70: register int i;
71:
72: if ((sdp = s->nextfree) != NULL)
73: s->nextfree = sdp->sdi_next;
74: else {
75: sdp = (struct sdi *)calloc(s->allocsize,sizeof(struct sdi));
76: if (sdp == NULL) sys_error("%s storage exceeded\n", s->segname);
77: sdp->sdi_next = s->head;
78: s->head = sdp++;
79: sdi_free = NULL;
80: for (i = s->allocsize-2; i--;) {
81: sdp->sdi_next = sdi_free;
82: sdi_free = sdp++;
83: }
84: s->nextfree = sdi_free;
85: s->allocsize *= 2;
86: }
87: s->nalloced++;
88: return(sdp);
89: }
90:
91: void
92: sdi_free(s)
93: register struct seglist *s;
94: {
95: register struct sdi *t, *u;
96: u = s->head;
97: while (u != NULL){
98: t = u->sdi_next;
99: free(u);
100: u = t;
101: }
102: s->head = NULL;
103: s->nextfree = NULL;
104: s->allocsize = SEGALLOC;
105: s->nalloced = 0;
106: }
107:
108: /*
109: * short-lived data structure
110: * used just for sorting the destination addresses, and keeping pointers
111: * to the corresponding sdi's.
112: */
113: struct sdi_destpair { struct sdi *user; int address; };
114:
115: int * destlist;
116: int destsize;
117:
118: sort_destpair( a, b )
119: struct sdi_destpair *a, *b;
120: {
121: return a->address - b->address;
122: }
123:
124: /*
125: * reformulate the base/span representation as follows:
126: * for each sdi, form a pair: a pointer to the sdi structure and
127: * the destination address (relative to the text base).
128: * Sort on destination address, then re-form as an integer array
129: * into which point the sdi structures. The array can be segmented
130: * to speed up processing it.
131: */
132: make_destlist()
133: {
134: struct sdi_destpair *darray;
135: register struct sdi_destpair *dp;
136: register struct sdi *s, *d;
137: register int *a;
138: register ndest = 0;
139: register lastaddr, newaddr;
140: int ddidelta;
141:
142: if ( sdilist.nalloced==0 ) return; /* empty list */
143: if ((darray = (struct sdi_destpair *)calloc( sdilist.nalloced , sizeof( struct sdi_destpair))) == NULL)
144: sys_error("cannot allocate intermediate structure");
145: for (s=sdi_list, dp = darray; s; s = s->sdi_next){
146: if (s->sdi_loc == -1 ) continue;
147: if (s->sdi_sym == NULL){
148: dp->address = s->sdi_off;
149: dp++ -> user = s;
150: ndest++;
151: }
152: }
153: if (destsize=ndest){
154: qsort( darray, ndest, sizeof (struct sdi_destpair), sort_destpair );
155: destlist = a = (int *)calloc( ndest, sizeof a[0] );
156: if (a==NULL)
157: sys_error("cannot allocate intermediate structure");
158: dp = darray;
159: *a = lastaddr = dp->address;
160: dp++ ->user->sdi_address = a++;
161: while (--ndest > 0){
162: if (lastaddr == (newaddr = dp->address)){
163: /* pick up the occasional duplicate destination address */
164: a--;
165: destsize--;
166: }else
167: *a = lastaddr = newaddr;
168: dp++ ->user->sdi_address = a++;
169: }
170: }
171: free( darray );
172: /*
173: * this is the part I hate.
174: * sweep through the sdi list and the dest list in parallel.
175: * for each sdi that was a ddi, subtract 2 from all destinations
176: * above the base address of the s(d)di. At the same time,
177: * sweep through the ddi list and subtract the shrinkage size
178: * from all destinations above the base address of the ddi.
179: * These are corrective factors we must apply because of initially
180: * assuming that all ddi's would be long. The base addresses of
181: * the sdi's have already been fixed by the routine resolve_ddi.
182: */
183: if (destsize){
184: a = destlist;
185: ndest = destsize;
186: s = sdi_list;
187: d = ddi_list;
188: ddidelta = 0;
189: while(ndest--){
190: newaddr = *a;
191: while( s != NULL && s->sdi_loc < newaddr ){
192: if (s->sdi_flags & SDI_DDI)
193: ddidelta += 2;
194: s = s->sdi_next;
195: }
196: while( d != NULL && d->sdi_loc < newaddr ){
197: ddidelta += d->sdi_flags;
198: d = d->sdi_next;
199: }
200: *a++ -= ddidelta;
201: }
202: }
203: }
204:
205: /*
206: * routine to create a sdi descriptor and insert it into the list
207: */
208: makesdi(op, base, flavor)
209: struct oper *op;/* the operand of the sdi */
210: long int base; /* origin of the the initial span of the sdi, to be */
211: /* corrected between pass1 and pass2 by subtracting */
212: /* from the value of the symbol of the operand */
213: /* i.e. 0 for resolving short absolute address modes */
214: /* Dot[+increment] for PC relative addresses */
215: {
216: register struct sdi *s, **p;
217: register int loc;
218: static long lastloc = INFINITY; /* you wanna see hackery? */
219: static struct sdi * lastsdi; /* we got hackery */
220:
221: if (!op ){ /* put the procedure boundary: .proc */
222: if (Oflag) return;
223: s = sdi_alloc(&sdilist);
224: s->sdi_loc = -1;
225: for ( p = lastsdi ? &(lastsdi->sdi_next) : &sdi_list;
226: *p;
227: p = &(*p)->sdi_next );
228: s->sdi_next = *p;
229: *p = s;
230: return;
231: }
232: if (op->flags_o&O_COMPLEX) /* not a simple address */
233: return(0);
234: s = sdi_alloc(&sdilist);
235: s->sdi_loc = dot; /* future bug here */
236: s->sdi_flags = flavor;
237: s->sdi_sym = op->sym_o;
238: s->sdi_off = op->value_o-op->sym_o->value_s; /* collect offset-from-label */
239: s->sdi_base = base;
240: s->sdi_leng = 2; /* shortest length */
241: # if EBUG
242: s->sdi_lineno = line_no;
243: # endif EBUG
244: loc = dot;
245: for ( p = (loc < lastloc) ? &sdi_list : &(lastsdi->sdi_next);
246: *p;
247: p = &(*p)->sdi_next)
248: if (loc < (*p)->sdi_loc)
249: break;
250: s->sdi_next = *p;
251: *p = s;
252: lastloc = loc;
253: lastsdi = s;
254: return(2); /* return the current length */
255: }
256:
257: /*
258: * resolve sdi's between pass1 and pass2
259: * basic algorithm is to repeatedly look for sdi that must use the
260: * long form, and update the span of other sdi's.
261: * When this terminates, all remaining sdi's can use the short form
262: */
263: sdi_resolve()
264: {
265: register struct sdi *s;
266: register struct sym_bkt *p;
267: register int t;
268: struct sdi *first, *end, *q;
269: long loc, big_delta = 0;
270: int *dest, *start;
271: int changed;
272: int csect_offsets[5];
273:
274: csect_offsets[C_UNDEF] = 0;
275: csect_offsets[C_TEXT ] = 0;
276: csect_offsets[C_DATA ] = tsize;
277: csect_offsets[C_DATA1] = tsize + dsize;
278: csect_offsets[C_DATA2] = tsize + dsize + d1size;
279:
280: for (s = sdi_list; s && s->sdi_loc == -1; s = s->sdi_next );
281: sdi_list = s; /* get rid of .proc at the beginning */
282:
283: for (s = sdi_list; s; s = s->sdi_next) {
284: if (s->sdi_loc == -1) continue;
285: p = s->sdi_sym;
286: if (SQUID(p->csect_s) && (p->attr_s & S_DEF)){
287: s->sdi_off += p->value_s + csect_offsets[p->csect_s];
288: s->sdi_sym = NULL;
289: } else
290: s->sdi_flags |= SDI_EXTERN;
291: }
292: ddi_resolve();
293: make_destlist();
294:
295: for (first = sdi_list; first; ) {
296: for (s=first; s && s->sdi_loc != -1; s=s->sdi_next )
297: end = s; /* find last sdi in this procedure */
298: for (s=end->sdi_next; s && s->sdi_loc == -1; s=s->sdi_next );
299: if (s)
300: loc = s->sdi_loc + total_delta;
301: else
302: loc = INFINITY;
303: if (destsize) {
304: for (dest = destlist;
305: *dest <= loc && dest < destlist + destsize;
306: dest++ );
307: start = --dest; /* last label before 'end' */
308: }
309: do {
310: changed = 0;
311: for (s = first; s && s->sdi_loc!=-1 ; s = s->sdi_next) {
312: if ((t = sdi_len(s) - s->sdi_leng) > 0) {
313: big_delta += t;
314: longsdi(s, t, first, start);
315: changed = 1;
316: } else if (t < 0) {
317: sys_error("Pathological sdi\n");
318: }
319: }
320: } while (changed);
321:
322: for (s=end->sdi_next; s && s->sdi_loc == -1; s=s->sdi_next );
323: first = s;
324: end->sdi_next = first; /* get rid of boundary .proc */
325: for (q = first; q; q = q->sdi_next )
326: q->sdi_base += big_delta;
327: if (destsize)
328: for ( dest = destlist + destsize - 1; dest > start; dest-- )
329: *dest += big_delta;
330: total_delta += big_delta;
331: big_delta = 0;
332: }
333: }
334:
335: /*
336: * update sdi list assuming the specified sdi must be long
337: */
338: longsdi(s, delta, first, start )
339: register struct sdi *s, *first;
340: register int delta;
341: int * start;
342: {
343: register struct sdi *t;
344: register long loc;
345: register int *dest;
346:
347: s->sdi_leng += delta; /* update the length of this sdi */
348: /* get the real location of the sdi */
349: loc = s->sdi_loc + sdi_inc( s->sdi_loc, first);
350: for (t = s->sdi_next; t && t->sdi_loc != -1; t = t->sdi_next)
351: t->sdi_base += delta;
352: if (destsize){
353: for ( dest = start; *dest > loc && dest >= destlist; dest-- )
354: *dest += delta;
355: }
356: }
357:
358: /*
359: * compute the length of the specified sdi by looking at the possible choices
360: */
361: sdi_len(s)
362: register struct sdi *s;
363: {
364: struct bchoice { int len; int lbound, ubound; };
365: static struct bchoice bsdi68[] = {
366: { 2, -128, 127 },
367: { 4, -32768, 32767 },
368: { 0, 0, 0 },
369: };
370: static struct bchoice bsdi4[] = {
371: { 2, -128, 127 },
372: { 0, 0, 0 },
373: };
374: static struct bchoice bsdip[] = {
375: { 2, -32768, 32767 },
376: { 0, 0, 0 },
377: };
378: /* SDI4, SDI6, SDI8, SDIP, */
379: static struct bchoice *blists[] = { bsdi4, bsdi68, bsdi68, bsdip, };
380: /* SDIX and SDIL are never span-dependent, so are not in this table! */
381:
382: register struct bchoice *b;
383: register span;
384:
385: span = *s->sdi_address - s->sdi_base;
386: if (!(s->sdi_flags&SDI_EXTERN)){
387: for (b = blists[ s->sdi_flags&SDI_TYPE ]; b->len; b++ )
388: if ( b->lbound <= span && span <= b->ubound)
389: return(b->len);
390: }
391: /* only the most general case will do */
392: return(glen[s->sdi_flags&SDI_TYPE]);
393: }
394:
395: /*
396: * return the total number of extra bytes due to long sdi's before
397: * the specified offset
398: */
399: sdi_inc(offset, first)
400: struct sdi *first;
401: register long offset;
402: {
403: register struct sdi *s;
404: register int total;
405:
406: if (first) {
407: s = first;
408: total = total_delta;
409: } else {
410: s = sdi_list;
411: total = 0;
412: }
413: for (; s; s = s->sdi_next) {
414: if (s->sdi_loc == -1 ) continue;
415: if (offset <= s->sdi_loc)
416: break;
417: total += s->sdi_leng - 2;
418: if (s->sdi_flags&SDI_DDI){
419: total -= 2;
420: }
421: }
422: for (s = ddi_list ; s; s = s->sdi_next) {
423: if (offset <= s->sdi_loc)
424: break;
425: total -= s->sdi_flags; /* if its still on the list, it shrank */
426: }
427: return(total);
428: }
429:
430: /*
431: * We are given an array of pointers to symbol-table entries for
432: * text labels, sorted by location. Update these by the changes in the
433: * sdi's. This used to be done by repeated calls to sdi_inc(), but this
434: * was obscenely expensive. Now that we have the lists sorted the same way,
435: * we should be able to compute this incrementally. Of course, there was
436: * the extra overhead of sorting the symbol list in the first place.
437: */
438: sdi_sym_update( symp, nsyms )
439: register struct sym_bkt **symp;
440: register int nsyms;
441: {
442: register struct sdi *sdi;
443: register struct sdi *ddi;
444: register curloc, curdelta;
445: register symloc, curddi;
446: curdelta = 0;
447:
448: if (nsyms==0) return;
449: if ((sdi = sdi_list) == NULL)
450: curloc = INFINITY;
451: else
452: curloc = sdi->sdi_loc;
453: if ((ddi = ddi_list) == NULL)
454: curddi = INFINITY;
455: else
456: curddi = ddi->sdi_loc;
457: while (nsyms--){
458: symloc = (*symp)->value_s;
459: while (symloc > curloc){
460: curdelta += sdi->sdi_leng - 2;
461: if (sdi->sdi_flags&SDI_DDI){
462: curdelta -= 2;
463: }
464: if ((sdi = sdi->sdi_next) == NULL){
465: curloc = INFINITY;
466: } else {
467: curloc = sdi->sdi_loc;
468: }
469: }
470: while (symloc > curddi){
471: curdelta -= ddi->sdi_flags ; /* if on ddi list, it shrank */
472: if ((ddi = ddi->sdi_next) == NULL){
473: curddi = INFINITY;
474: } else {
475: curddi = ddi->sdi_loc;
476: }
477: }
478: (*symp++)->value_s = symloc + curdelta;
479: }
480: }
481:
482: /*
483: * Data-dependent instructions
484: * are those whose length we may not be able to ascertain at first pass,
485: * but which we would like to address in the most effecient manner.
486: * Determining their length could, in full generality, be an iterative
487: * process, but I think we can make some useful simplifications that
488: * will do most of what we desire in one pass over our intermediate
489: * data structure. The gross structure will mimic the sdi code above.
490: * The instructions we are dealing with here have three forms:
491: * long absolute, word absolute, and PC-relative. Thus, there are
492: * two lengths of addressing extensions: two bytes and four bytes.
493: */
494:
495: /*
496: * Statistically, ALL data references are long. Thus it makes little
497: * sense to follow the sdi strategy of assuming the shortest form until
498: * proven otherwise. Since the ddi list is only looked at once, we
499: * don't have to maintain the data structures required by the sdi algorithm.
500: * In fact, once we've decided that a reference is long, we can take it
501: * entirely off the list, if we make this assumption.
502: */
503: makeddi(op, base, flavor, wrtflg )
504: struct oper *op;/* the operand of the sdi */
505: long int base; /* origin of the the initial span of the sdi, to be */
506: /* corrected between pass1 and pass2 by subtracting */
507: /* from the value of the symbol of the operand */
508: /* i.e. 0 for resolving short absolute address modes */
509: /* Dot[+increment] for PC relative addresses */
510: int flavor; /* form of ddi: usually SDIP, but for index SDIX, or SDIL for link*/
511: int wrtflg; /* 0 => not memory write destination: PC-relative ok */
512: /* 1 => memory write destination: PC-relative NOT ok */
513: {
514: register struct sdi *s, **p;
515: register int loc;
516: static long lastloc = INFINITY; /* you wanna see hackery? */
517: static struct sdi * lastddi; /* we got hackery */
518:
519: if (op->flags_o&O_COMPLEX) /* not a simple address */
520: return(0);
521: s = sdi_alloc( &ddilist );
522: s->sdi_loc = dot; /* soon to be a bug */
523: s->sdi_sym = op->sym_o;
524: s->sdi_off = op->value_o;
525: s->sdi_base = base;
526: s->sdi_leng = 4; /* assumed length */
527: s->sdi_flags = flavor;
528: # if EBUG
529: s->sdi_lineno = line_no;
530: # endif EBUG
531: if (wrtflg) s->sdi_flags |= SDI_DDI;
532: loc = dot;
533:
534: for ( p = (loc < lastloc) ? &ddi_list : &(lastddi->sdi_next);
535: *p;
536: p = &(*p)->sdi_next)
537: if (loc < (*p)->sdi_loc)
538: break;
539: s->sdi_next = *p;
540: lastloc = loc;
541: lastddi = s;
542: *p = s;
543: return(flavor==SDI6?6:4); /* return the current length */
544: }
545:
546:
547: ddi_resolve()
548: {
549: register struct sdi *s, **spp, *q;
550: register struct sym_bkt *p;
551: register int t;
552:
553: spp = &ddi_list;
554: while (s = *spp){
555: p = s->sdi_sym;
556: switch (s->sdi_flags){
557: case SDIP:
558: if (SQUID(p->csect_s ) && (p->attr_s & S_DEF) ){
559: /* this looks like a job for PC@() */
560: /* fill out the sdi structure, and
561: insert it in the sdi list */
562: register struct sdi **pp;
563: struct sdi *newsdi;
564:
565: s->sdi_off += p->value_s;
566: s->sdi_sym = NULL;
567: /* take ourselves off of ddi list */
568: *spp = s->sdi_next;
569: /*
570: * here's the tricky part:
571: * all the addresses in our lists assume this
572: * address has length four. We will now change
573: * them so it appears to have length two.
574: */
575: shortddi( s, 2 );
576: s->sdi_flags = SDI_DDI|SDIP;
577: /* now put ourselves in the sdi list */
578: for (pp = &sdi_list; *pp; pp = &(*pp)->sdi_next)
579: if (s->sdi_loc < (*pp)->sdi_loc)
580: break;
581: newsdi = sdi_alloc(&sdilist);
582: *newsdi = *s;
583: newsdi->sdi_next = *pp;
584: *pp = newsdi;
585: /* fix addresses on sdi list, too */
586: newsdi->sdi_leng -= 2;
587: for ( q= newsdi->sdi_next; q; q = q->sdi_next )
588: q->sdi_base -= 2;
589: /* sdi routines will fix it up later */
590: continue;
591: }
592: /* else fall through */
593: case SDIP|SDI_DDI:
594: case SDI4:
595: case SDI4|SDI_DDI:
596: if ((p->attr_s & S_DEF) && p->csect_s == C_UNDEF) {
597: /* try on a short absolute */
598: t = p->value_s + s->sdi_off;
599: if (t >= -32768 && t <= 32767 ){
600: shortddi( s, 2 );
601: spp = &s->sdi_next;
602: continue;
603: }
604:
605: }
606: /* else it does not shrink -- take off list */
607: break;
608: case SDI6:
609: case SDI6|SDI_DDI:
610: /*
611: * Index mode.
612: */
613: /* Write bit only confuses us: delete it */
614: s->sdi_flags = SDI6;
615: if ((p->attr_s & S_DEF) && p->csect_s == C_UNDEF){
616: /* try very short, then just plain short */
617: t = p->value_s + s->sdi_off;
618: if ( t >= -128 && t <= 127 ){
619: /* shrinks much */
620: shortddi( s, 4 );
621: spp = &s->sdi_next;
622: continue;
623: } else if (t >= -32768 && t <= 32767 ){
624: /* shrinks some */
625: shortddi( s, 2 );
626: spp = &s->sdi_next;
627: continue;
628: }
629: }
630: /* else it does not shrink -- take off list */
631: break;
632: case SDIX:
633: case SDIX|SDI_DDI:
634: /*
635: * should be displacement mode, but will be
636: * index mode with supressed indexing if the
637: * displacement is too big.
638: */
639: /* FALL THROUGH */
640: case SDIL:
641: /*
642: * link instruction: looks like SDIP, but is not
643: * PC-relative.
644: */
645: /* Write bit only confuses us: delete it */
646: s->sdi_flags &= SDI_TYPE;
647: if ((p->attr_s & S_DEF) && p->csect_s == C_UNDEF){
648: /* try short */
649: t = p->value_s + s->sdi_off;
650: if (t >= -32768 && t <= 32767 ){
651: /* shrinks */
652: shortddi( s, glen[s->sdi_flags&SDI_TYPE] - 2 );
653: spp = &s->sdi_next;
654: continue;
655: }
656: }
657: /* else it does not shrink -- take off list */
658: break;
659: }
660: /* take ddi off the list */
661: *spp = s->sdi_next;
662: }
663: if (ddi_list == NULL){
664: sdi_free(&ddilist); /* all gone */
665: } else {
666: register int cursdi;
667: /*
668: * Sweep through ddi list and sdi list in parallel,
669: * fixing up sdi base-addresses.
670: */
671: s = ddi_list;
672: t = 0;
673: q = sdi_list;
674: while (q != NULL){
675: cursdi = q->sdi_base;
676: while ( s != NULL && s->sdi_base < cursdi ){
677: t += s->sdi_flags;
678: s = s->sdi_next;
679: }
680: q->sdi_base -= t;
681: q = q->sdi_next;
682: }
683: }
684: }
685:
686: shortddi( s, d )
687: register struct sdi *s;
688: register d;
689: {
690: /*
691: * A ddi shrank! Adjust all lists accordingly.
692: */
693: register struct sdi *t;
694: register loc = s->sdi_loc+sdi_inc( s->sdi_loc, NULL);
695:
696: s->sdi_flags = d; /* how much I shrank */
697: for (t = ddi_list; t; t = t->sdi_next)
698: if (t != s && t->sdi_base > loc)
699: t->sdi_base -= d;
700: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.