|
|
1.1 ! root 1: /* Generate code from to output assembler insns as recognized from rtl. ! 2: Copyright (C) 1987 Free Software Foundation, Inc. ! 3: ! 4: This file is part of GNU CC. ! 5: ! 6: GNU CC is distributed in the hope that it will be useful, ! 7: but WITHOUT ANY WARRANTY. No author or distributor ! 8: accepts responsibility to anyone for the consequences of using it ! 9: or for whether it serves any particular purpose or works at all, ! 10: unless he says so in writing. Refer to the GNU CC General Public ! 11: License for full details. ! 12: ! 13: Everyone is granted permission to copy, modify and redistribute ! 14: GNU CC, but only under the conditions described in the ! 15: GNU CC General Public License. A copy of this license is ! 16: supposed to have been given to you along with GNU CC so you ! 17: can know your rights and responsibilities. It should be in a ! 18: file named COPYING. Among other things, the copyright notice ! 19: and this notice must be preserved on all copies. */ ! 20: ! 21: ! 22: /* This program reads the machine description for the compiler target machine ! 23: and produces a file containing three things: ! 24: ! 25: 1, An array of strings `insn_template' which is indexed by insn code number ! 26: and contains the template for output of that insn, ! 27: ! 28: 2. An array of ints `insn_n_operands' which is indexed by insn code number ! 29: and contains the number of distinct operands in the pattern for that insn, ! 30: ! 31: 3. An array of ints `insn_n_dups' which is indexed by insn code number ! 32: and contains the number of match_dup's that appear in the insn's pattern. ! 33: This says how many elements of `recog_dup_loc' are significant ! 34: after an insn has been recognized. ! 35: ! 36: 4. An array of arrays of operand constraint strings, ! 37: `insn_operand_constraint', ! 38: indexed first by insn code number and second by operand number, ! 39: containing the constraint for that operand. ! 40: ! 41: This array is generated only if register constraints appear in ! 42: match_operand rtx's. ! 43: ! 44: 5. An array of arrays of chars which indicate which operands of ! 45: which insn patterns appear within ADDRESS rtx's. This array is ! 46: called `insn_operand_address_p' and is generated only if there ! 47: are *no* register constraints in the match_operand rtx's. ! 48: ! 49: 6. An array of arrays of machine modes, `insn_operand_mode', ! 50: indexed first by insn code number and second by operand number, ! 51: containing the machine mode that that operand is supposed to have. ! 52: Also `insn_operand_strict_low', which is nonzero for operands ! 53: contained in a STRICT_LOW_PART. ! 54: ! 55: 7. An array of arrays of int-valued functions, `insn_operand_predicate', ! 56: indexed first by insn code number and second by operand number, ! 57: containing the match_operand predicate for this operand. ! 58: ! 59: 8. An array of functions `insn_gen_function' which, indexed ! 60: by insn code number, gives the function to generate a body ! 61: for that patter, given operands as arguments. ! 62: ! 63: 9. A function `output_insn_hairy' which is called with two arguments ! 64: (an insn code number and a vector of operand value rtx's) ! 65: and returns a template to use for output of that insn. ! 66: This is used only in the cases where the template is not constant. ! 67: These cases are specified by a * at the beginning of the template string ! 68: in the machine description. They are identified for the sake of ! 69: other parts of the compiler by a zero element in `insn_template'. ! 70: ! 71: The code number of an insn is simply its position in the machine description; ! 72: code numbers are assigned sequentially to entries in the description, ! 73: starting with code number 0. ! 74: ! 75: Thus, the following entry in the machine description ! 76: ! 77: (define_insn "clrdf" ! 78: [(set (match_operand:DF 0 "general_operand" "") ! 79: (const_int 0))] ! 80: "" ! 81: "clrd %0") ! 82: ! 83: assuming it is the 25th entry present, would cause ! 84: insn_template[24] to be "clrd %0", and insn_n_operands[24] to be 1. ! 85: It would not make an case in output_insn_hairy because the template ! 86: given in the entry is a constant (it does not start with `*'). */ ! 87: ! 88: #include <stdio.h> ! 89: #include "config.h" ! 90: #include "rtl.h" ! 91: #include "obstack.h" ! 92: ! 93: /* No instruction can have more operands than this. ! 94: Sorry for this arbitrary limit, but what machine will ! 95: have an instruction with this many operands? */ ! 96: ! 97: #define MAX_MAX_OPERANDS 40 ! 98: ! 99: struct obstack obstack; ! 100: struct obstack *rtl_obstack = &obstack; ! 101: ! 102: #define obstack_chunk_alloc xmalloc ! 103: #define obstack_chunk_free free ! 104: extern int xmalloc (); ! 105: extern void free (); ! 106: ! 107: void fatal (); ! 108: void mybcopy (); ! 109: void mybzero (); ! 110: ! 111: /* insns in the machine description are assigned sequential code numbers ! 112: that are used by insn-recog.c (produced by genrecog) to communicate ! 113: to insn-output.c (produced by this program). */ ! 114: ! 115: int next_code_number; ! 116: ! 117: /* Record in this chain all information that we will output, ! 118: associated with the code number of the insn. */ ! 119: ! 120: struct data ! 121: { ! 122: int code_number; ! 123: char *name; ! 124: char *template; /* string such as "movl %1,%0" */ ! 125: int n_operands; /* Number of operands this insn recognizes */ ! 126: int n_dups; /* Number times match_dup appears in pattern */ ! 127: struct data *next; ! 128: char *constraints[MAX_MAX_OPERANDS]; ! 129: char *predicates[MAX_MAX_OPERANDS]; ! 130: char address_p[MAX_MAX_OPERANDS]; ! 131: enum machine_mode modes[MAX_MAX_OPERANDS]; ! 132: char strict_low[MAX_MAX_OPERANDS]; ! 133: char outfun; /* Nonzero means this has an output function */ ! 134: }; ! 135: ! 136: /* This variable points to the first link in the chain. */ ! 137: ! 138: struct data *insn_data; ! 139: ! 140: /* Pointer to the last link in the chain, so new elements ! 141: can be added at the end. */ ! 142: ! 143: struct data *end_of_insn_data; ! 144: ! 145: /* Nonzero if any match_operand has a constraint string; ! 146: implies that REGISTER_CONSTRAINTS will be defined ! 147: for this machine description. */ ! 148: ! 149: int have_constraints; ! 150: ! 151: void ! 152: output_prologue () ! 153: { ! 154: ! 155: printf ("/* Generated automatically by the program `genoutput'\n\ ! 156: from the machine description file `md'. */\n\n"); ! 157: ! 158: printf ("#include \"config.h\"\n"); ! 159: printf ("#include \"rtl.h\"\n"); ! 160: printf ("#include \"regs.h\"\n"); ! 161: printf ("#include \"conditions.h\"\n"); ! 162: printf ("#include \"insn-flags.h\"\n"); ! 163: printf ("#include \"insn-config.h\"\n\n"); ! 164: printf ("#include \"output.h\"\n"); ! 165: ! 166: printf ("#include \"aux-output.c\"\n\n"); ! 167: } ! 168: ! 169: void ! 170: output_epilogue () ! 171: { ! 172: register struct data *d; ! 173: ! 174: printf ("\nchar *insn_template[] =\n {\n"); ! 175: for (d = insn_data; d; d = d->next) ! 176: { ! 177: if (d->template) ! 178: printf (" \"%s\",\n", d->template); ! 179: else ! 180: printf (" 0,\n"); ! 181: } ! 182: printf (" };\n"); ! 183: ! 184: printf ("\nchar *(*insn_outfun[])() =\n {\n"); ! 185: for (d = insn_data; d; d = d->next) ! 186: { ! 187: if (d->outfun) ! 188: printf (" output_%d,", d->code_number); ! 189: else ! 190: printf (" 0,\n"); ! 191: } ! 192: printf (" };\n"); ! 193: ! 194: printf ("\nrtx (*insn_gen_function[]) () =\n {\n"); ! 195: for (d = insn_data; d; d = d->next) ! 196: { ! 197: if (d->name) ! 198: printf (" gen_%s,\n", d->name); ! 199: else ! 200: printf (" 0,\n"); ! 201: } ! 202: printf (" };\n"); ! 203: ! 204: printf ("\nint insn_n_operands[] =\n {\n"); ! 205: for (d = insn_data; d; d = d->next) ! 206: { ! 207: printf (" %d,\n", d->n_operands); ! 208: } ! 209: printf (" };\n"); ! 210: ! 211: printf ("\nint insn_n_dups[] =\n {\n"); ! 212: for (d = insn_data; d; d = d->next) ! 213: { ! 214: printf (" %d,\n", d->n_dups); ! 215: } ! 216: printf (" };\n"); ! 217: ! 218: if (have_constraints) ! 219: { ! 220: printf ("\nchar *insn_operand_constraint[][MAX_RECOG_OPERANDS] =\n {\n"); ! 221: for (d = insn_data; d; d = d->next) ! 222: { ! 223: register int i; ! 224: printf (" {"); ! 225: for (i = 0; i < d->n_operands; i++) ! 226: { ! 227: if (d->constraints[i] == 0) ! 228: printf (" \"\","); ! 229: else ! 230: printf (" \"%s\",", d->constraints[i]); ! 231: } ! 232: if (d->n_operands == 0) ! 233: printf (" 0"); ! 234: printf (" },\n"); ! 235: } ! 236: printf (" };\n"); ! 237: } ! 238: else ! 239: { ! 240: printf ("\nchar insn_operand_address_p[][MAX_RECOG_OPERANDS] =\n {\n"); ! 241: for (d = insn_data; d; d = d->next) ! 242: { ! 243: register int i; ! 244: printf (" {"); ! 245: for (i = 0; i < d->n_operands; i++) ! 246: printf (" %d,", d->address_p[i]); ! 247: if (d->n_operands == 0) ! 248: printf (" 0"); ! 249: printf (" },\n"); ! 250: } ! 251: printf (" };\n"); ! 252: } ! 253: ! 254: printf ("\nenum machine_mode insn_operand_mode[][MAX_RECOG_OPERANDS] =\n {\n"); ! 255: for (d = insn_data; d; d = d->next) ! 256: { ! 257: register int i; ! 258: printf (" {"); ! 259: for (i = 0; i < d->n_operands; i++) ! 260: printf (" %smode,", GET_MODE_NAME (d->modes[i])); ! 261: if (d->n_operands == 0) ! 262: printf (" VOIDmode"); ! 263: printf (" },\n"); ! 264: } ! 265: printf (" };\n"); ! 266: ! 267: printf ("\nchar insn_operand_strict_low[][MAX_RECOG_OPERANDS] =\n {\n"); ! 268: for (d = insn_data; d; d = d->next) ! 269: { ! 270: register int i; ! 271: printf (" {"); ! 272: for (i = 0; i < d->n_operands; i++) ! 273: printf (" %d,", d->strict_low[i]); ! 274: if (d->n_operands == 0) ! 275: printf (" 0"); ! 276: printf (" },\n"); ! 277: } ! 278: printf (" };\n"); ! 279: ! 280: printf ("\nint (*insn_operand_predicate[][MAX_RECOG_OPERANDS])() =\n {\n"); ! 281: for (d = insn_data; d; d = d->next) ! 282: { ! 283: register int i; ! 284: printf (" {"); ! 285: for (i = 0; i < d->n_operands; i++) ! 286: printf (" %s,", ((d->predicates[i] && d->predicates[i][0]) ! 287: ? d->predicates[i] : "0")); ! 288: if (d->n_operands == 0) ! 289: printf (" 0"); ! 290: printf (" },\n"); ! 291: } ! 292: printf (" };\n"); ! 293: } ! 294: ! 295: /* scan_operands (X) stores in max_opno the largest operand ! 296: number present in X, if that is larger than the previous ! 297: value of max_opno. It stores all the constraints in `constraints' ! 298: and all the machine modes in `modes'. ! 299: ! 300: THIS_ADDRESS_P is nonzero if the containing rtx was an ADDRESS. ! 301: THIS_STRICT_LOW is nonzero if the containing rtx was a STRICT_LOW_PART. */ ! 302: ! 303: int max_opno; ! 304: int num_dups; ! 305: char *constraints[MAX_MAX_OPERANDS]; ! 306: char *predicates[MAX_MAX_OPERANDS]; ! 307: char address_p[MAX_MAX_OPERANDS]; ! 308: enum machine_mode modes[MAX_MAX_OPERANDS]; ! 309: char strict_low[MAX_MAX_OPERANDS]; ! 310: ! 311: void ! 312: scan_operands (part, this_address_p, this_strict_low) ! 313: rtx part; ! 314: int this_address_p; ! 315: int this_strict_low; ! 316: { ! 317: register int i, j; ! 318: register RTX_CODE code; ! 319: register char *format_ptr; ! 320: ! 321: if (part == 0) ! 322: return; ! 323: ! 324: code = GET_CODE (part); ! 325: ! 326: if (code == MATCH_OPERAND) ! 327: { ! 328: int opno = XINT (part, 0); ! 329: if (opno > max_opno) ! 330: max_opno = opno; ! 331: if (max_opno >= MAX_MAX_OPERANDS) ! 332: fatal ("Too many operands (%d) in one instruction pattern.\n", ! 333: max_opno + 1); ! 334: modes[opno] = GET_MODE (part); ! 335: strict_low[opno] = this_strict_low; ! 336: predicates[opno] = XSTR (part, 1); ! 337: constraints[opno] = XSTR (part, 2); ! 338: if (XSTR (part, 2) != 0 && *XSTR (part, 2) != 0) ! 339: have_constraints = 1; ! 340: address_p[opno] = this_address_p; ! 341: return; ! 342: } ! 343: ! 344: if (code == MATCH_DUP) ! 345: { ! 346: ++num_dups; ! 347: return; ! 348: } ! 349: ! 350: if (code == ADDRESS) ! 351: { ! 352: scan_operands (XEXP (part, 0), 1, 0); ! 353: return; ! 354: } ! 355: ! 356: if (code == STRICT_LOW_PART) ! 357: { ! 358: scan_operands (XEXP (part, 0), 0, 1); ! 359: return; ! 360: } ! 361: ! 362: format_ptr = GET_RTX_FORMAT (GET_CODE (part)); ! 363: ! 364: for (i = 0; i < GET_RTX_LENGTH (GET_CODE (part)); i++) ! 365: switch (*format_ptr++) ! 366: { ! 367: case 'e': ! 368: scan_operands (XEXP (part, i), 0, 0); ! 369: break; ! 370: case 'E': ! 371: if (XVEC (part, i) != NULL) ! 372: for (j = 0; j < XVECLEN (part, i); j++) ! 373: scan_operands (XVECEXP (part, i, j), 0, 0); ! 374: break; ! 375: } ! 376: } ! 377: ! 378: /* Look at a define_insn just read. Assign its code number. ! 379: Record on insn_data the template and the number of arguments. ! 380: If the insn has a hairy output action, output a function for now. */ ! 381: ! 382: void ! 383: gen_insn (insn) ! 384: rtx insn; ! 385: { ! 386: register struct data *d = (struct data *) xmalloc (sizeof (struct data)); ! 387: register int i; ! 388: ! 389: d->code_number = next_code_number++; ! 390: if (XSTR (insn, 0)[0]) ! 391: d->name = XSTR (insn, 0); ! 392: else ! 393: d->name = 0; ! 394: ! 395: /* Build up the list in the same order as the insns are seen ! 396: in the machine description. */ ! 397: d->next = 0; ! 398: if (end_of_insn_data) ! 399: end_of_insn_data->next = d; ! 400: else ! 401: insn_data = d; ! 402: ! 403: end_of_insn_data = d; ! 404: ! 405: max_opno = -1; ! 406: num_dups = 0; ! 407: ! 408: mybzero (constraints, sizeof constraints); ! 409: mybzero (predicates, sizeof predicates); ! 410: mybzero (address_p, sizeof address_p); ! 411: mybzero (modes, sizeof modes); ! 412: mybzero (strict_low, sizeof strict_low); ! 413: for (i = 0; i < XVECLEN (insn, 1); i++) ! 414: scan_operands (XVECEXP (insn, 1, i), 0, 0); ! 415: d->n_operands = max_opno + 1; ! 416: d->n_dups = num_dups; ! 417: mybcopy (constraints, d->constraints, sizeof constraints); ! 418: mybcopy (predicates, d->predicates, sizeof predicates); ! 419: mybcopy (address_p, d->address_p, sizeof address_p); ! 420: mybcopy (modes, d->modes, sizeof modes); ! 421: mybcopy (strict_low, d->strict_low, sizeof strict_low); ! 422: ! 423: /* We need to consider only the instructions whose assembler code template ! 424: starts with a *. These are the ones where the template is really ! 425: C code to run to decide on a template to use. ! 426: So for all others just return now. */ ! 427: ! 428: if (XSTR (insn, 3)[0] != '*') ! 429: { ! 430: d->template = XSTR (insn, 3); ! 431: d->outfun = 0; ! 432: return; ! 433: } ! 434: ! 435: d->template = 0; ! 436: d->outfun = 1; ! 437: ! 438: printf ("\nchar *\n"); ! 439: printf ("output_%d (operands, insn)\n", d->code_number); ! 440: printf (" rtx *operands;\n"); ! 441: printf (" rtx insn;\n"); ! 442: printf ("{\n"); ! 443: /* The following is done in a funny way to get around problems in ! 444: VAX-11 "C" on VMS. It is the equivalent of: ! 445: printf ("%s\n", &(XSTR (insn, 3)[1])); */ ! 446: { ! 447: register char *cp = &(XSTR (insn, 3)[1]); ! 448: while (*cp) putchar (*cp++); ! 449: putchar ('\n'); ! 450: } ! 451: printf ("}\n"); ! 452: } ! 453: ! 454: /* Look at a define_peephole just read. Assign its code number. ! 455: Record on insn_data the template and the number of arguments. ! 456: If the insn has a hairy output action, output it now. */ ! 457: ! 458: void ! 459: gen_peephole (peep) ! 460: rtx peep; ! 461: { ! 462: register struct data *d = (struct data *) xmalloc (sizeof (struct data)); ! 463: register int i; ! 464: ! 465: d->code_number = next_code_number++; ! 466: d->name = 0; ! 467: ! 468: /* Build up the list in the same order as the insns are seen ! 469: in the machine description. */ ! 470: d->next = 0; ! 471: if (end_of_insn_data) ! 472: end_of_insn_data->next = d; ! 473: else ! 474: insn_data = d; ! 475: ! 476: end_of_insn_data = d; ! 477: ! 478: max_opno = -1; ! 479: mybzero (constraints, sizeof constraints); ! 480: ! 481: /* Get the number of operands by scanning all the ! 482: patterns of the peephole optimizer. ! 483: But ignore all the rest of the information thus obtained. */ ! 484: for (i = 0; i < XVECLEN (peep, 0); i++) ! 485: scan_operands (XVECEXP (peep, 0, i), 0, 0); ! 486: ! 487: d->n_operands = max_opno + 1; ! 488: d->n_dups = 0; ! 489: mybcopy (constraints, d->constraints, sizeof constraints); ! 490: mybzero (d->predicates, sizeof predicates); ! 491: mybzero (d->address_p, sizeof address_p); ! 492: mybzero (d->modes, sizeof modes); ! 493: mybzero (d->strict_low, sizeof strict_low); ! 494: ! 495: /* We need to consider only the instructions whose assembler code template ! 496: starts with a *. These are the ones where the template is really ! 497: C code to run to decide on a template to use. ! 498: So for all others just return now. */ ! 499: ! 500: if (XSTR (peep, 2)[0] != '*') ! 501: { ! 502: d->template = XSTR (peep, 2); ! 503: d->outfun = 0; ! 504: return; ! 505: } ! 506: ! 507: d->template = 0; ! 508: d->outfun = 1; ! 509: ! 510: printf ("\nchar *\n"); ! 511: printf ("output_%d (operands, insn)\n", d->code_number); ! 512: printf (" rtx *operands;\n"); ! 513: printf (" rtx insn;\n"); ! 514: printf ("{\n"); ! 515: printf ("%s\n", &(XSTR (peep, 2)[1])); ! 516: printf ("}\n"); ! 517: } ! 518: ! 519: /* Process a define_expand just read. Assign its code number, ! 520: only for the purposes of `insn_gen_function'. */ ! 521: ! 522: void ! 523: gen_expand (insn) ! 524: rtx insn; ! 525: { ! 526: register struct data *d = (struct data *) xmalloc (sizeof (struct data)); ! 527: register int i; ! 528: ! 529: d->code_number = next_code_number++; ! 530: if (XSTR (insn, 0)[0]) ! 531: d->name = XSTR (insn, 0); ! 532: else ! 533: d->name = 0; ! 534: ! 535: /* Build up the list in the same order as the insns are seen ! 536: in the machine description. */ ! 537: d->next = 0; ! 538: if (end_of_insn_data) ! 539: end_of_insn_data->next = d; ! 540: else ! 541: insn_data = d; ! 542: ! 543: end_of_insn_data = d; ! 544: ! 545: max_opno = -1; ! 546: num_dups = 0; ! 547: ! 548: /* Scan the operands to get the specified predicates and modes, ! 549: since expand_binop needs to know them. */ ! 550: ! 551: mybzero (predicates, sizeof predicates); ! 552: mybzero (modes, sizeof modes); ! 553: if (XVEC (insn, 1)) ! 554: for (i = 0; i < XVECLEN (insn, 1); i++) ! 555: scan_operands (XVECEXP (insn, 1, i), 0, 0); ! 556: d->n_operands = max_opno + 1; ! 557: mybcopy (predicates, d->predicates, sizeof predicates); ! 558: mybcopy (modes, d->modes, sizeof modes); ! 559: ! 560: mybzero (d->constraints, sizeof constraints); ! 561: mybzero (d->address_p, sizeof address_p); ! 562: mybzero (d->strict_low, sizeof strict_low); ! 563: ! 564: d->n_dups = 0; ! 565: d->template = 0; ! 566: d->outfun = 0; ! 567: } ! 568: ! 569: int ! 570: xmalloc (size) ! 571: { ! 572: register int val = malloc (size); ! 573: ! 574: if (val == 0) ! 575: fatal ("virtual memory exhausted"); ! 576: return val; ! 577: } ! 578: ! 579: int ! 580: xrealloc (ptr, size) ! 581: char *ptr; ! 582: int size; ! 583: { ! 584: int result = realloc (ptr, size); ! 585: if (!result) ! 586: fatal ("virtual memory exhausted"); ! 587: return result; ! 588: } ! 589: ! 590: void ! 591: mybzero (b, length) ! 592: register char *b; ! 593: register int length; ! 594: { ! 595: while (length-- > 0) ! 596: *b++ = 0; ! 597: } ! 598: ! 599: void ! 600: mybcopy (b1, b2, length) ! 601: register char *b1; ! 602: register char *b2; ! 603: register int length; ! 604: { ! 605: while (length-- > 0) ! 606: *b2++ = *b1++; ! 607: } ! 608: ! 609: void ! 610: fatal (s, a1, a2) ! 611: { ! 612: fprintf (stderr, "genoutput: "); ! 613: fprintf (stderr, s, a1, a2); ! 614: fprintf (stderr, "\n"); ! 615: exit (FATAL_EXIT_CODE); ! 616: } ! 617: ! 618: int ! 619: main (argc, argv) ! 620: int argc; ! 621: char **argv; ! 622: { ! 623: rtx desc; ! 624: FILE *infile; ! 625: extern rtx read_rtx (); ! 626: register int c; ! 627: ! 628: obstack_init (rtl_obstack); ! 629: ! 630: if (argc <= 1) ! 631: fatal ("No input file name."); ! 632: ! 633: infile = fopen (argv[1], "r"); ! 634: if (infile == 0) ! 635: { ! 636: perror (argv[1]); ! 637: exit (FATAL_EXIT_CODE); ! 638: } ! 639: ! 640: init_rtl (); ! 641: ! 642: output_prologue (); ! 643: next_code_number = 0; ! 644: have_constraints = 0; ! 645: ! 646: /* Read the machine description. */ ! 647: ! 648: while (1) ! 649: { ! 650: c = read_skip_spaces (infile); ! 651: if (c == EOF) ! 652: break; ! 653: ungetc (c, infile); ! 654: ! 655: desc = read_rtx (infile); ! 656: if (GET_CODE (desc) == DEFINE_INSN) ! 657: gen_insn (desc); ! 658: if (GET_CODE (desc) == DEFINE_PEEPHOLE) ! 659: gen_peephole (desc); ! 660: if (GET_CODE (desc) == DEFINE_EXPAND) ! 661: gen_expand (desc); ! 662: } ! 663: ! 664: output_epilogue (); ! 665: ! 666: fflush (stdout); ! 667: exit (ferror (stdout) != 0 ? FATAL_EXIT_CODE : SUCCESS_EXIT_CODE); ! 668: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.