|
|
1.1 ! root 1: /* $Header: /newbits/kernel/USRSRC/coh/RCS/bio.c,v 1.4 91/07/24 07:49:45 bin Exp Locker: bin $ */ ! 2: /* (lgl- ! 3: * The information contained herein is a trade secret of Mark Williams ! 4: * Company, and is confidential information. It is provided under a ! 5: * license agreement, and may be copied or disclosed only under the ! 6: * terms of that agreement. Any reproduction or disclosure of this ! 7: * material without the express written authorization of Mark Williams ! 8: * Company or persuant to the license agreement is unlawful. ! 9: * ! 10: * COHERENT Version 2.3.37 ! 11: * Copyright (c) 1982, 1983, 1984. ! 12: * An unpublished work by Mark Williams Company, Chicago. ! 13: * All rights reserved. ! 14: -lgl) */ ! 15: /* ! 16: * Coherent. ! 17: * Buffered I/O. ! 18: * ! 19: * $Log: bio.c,v $ ! 20: * Revision 1.4 91/07/24 07:49:45 bin ! 21: * update prov by hal ! 22: * ! 23: * ! 24: * Revision 1.1 88/03/24 16:13:29 src ! 25: * Initial revision ! 26: * ! 27: * 87/11/25 Allan Cornish /usr/src/sys/coh/bio.c ! 28: * vaddr_t bp->b_vaddr --> faddr_t bp->b_faddr. ! 29: * ! 30: * 87/11/05 Allan Cornish /usr/src/sys/coh/bio.c ! 31: * New seg struct now used to allow extended addressing. ! 32: * ! 33: * 87/01/05 Allan Cornish /usr/src/sys/coh/bio.c ! 34: * ioreq() now only wakes &stimer if the swap timer is active. ! 35: * ! 36: * 86/12/12 Allan Cornish /usr/src/sys/coh/bio.c ! 37: * Added 3rd arg to dpoll() to specify blocking poll if non-zero. ! 38: * ! 39: * 86/11/19 Allan Cornish /usr/src/sys/coh/bio.c ! 40: * Added dpoll() routine to perform device polls [System V.3 compatible]. ! 41: * ! 42: * 86/07/24 Allan Cornish /usr/src/sys/coh/bio.c ! 43: * Added check in devinit() for null dp->d_conp->c_load function pointer. ! 44: */ ! 45: #include <sys/coherent.h> ! 46: #include <sys/buf.h> ! 47: #include <sys/con.h> ! 48: #include <errno.h> ! 49: #include <sys/io.h> ! 50: #include <sys/proc.h> ! 51: #include <sys/sched.h> ! 52: #include <sys/seg.h> ! 53: #include <sys/stat.h> ! 54: #include <sys/uproc.h> ! 55: ! 56: /* ! 57: * Initialise buffer headers. ! 58: */ ! 59: bufinit() ! 60: { ! 61: register BUF *bp; ! 62: faddr_t f; ! 63: paddr_t p; ! 64: ! 65: FP_SEL(f) = sds; ! 66: FP_OFF(f) = 0; ! 67: p = blockp; ! 68: FP_OFF(f) = blockp - vtop(f); ! 69: ! 70: bufl = kalloc(NBUF * sizeof(BUF)); ! 71: if (bufl == NULL) ! 72: panic("bufinit: no space for BUF's"); ! 73: ! 74: for (bp=&bufl[NBUF-1]; bp >= bufl; --bp) { ! 75: bp->b_dev = NODEV; ! 76: bp->b_faddr = f; ! 77: bp->b_paddr = p; ! 78: FP_OFF(f) += BSIZE; ! 79: p += BSIZE; ! 80: } ! 81: } ! 82: ! 83: /* ! 84: * Synchronise the buffer cache. ! 85: */ ! 86: bsync() ! 87: { ! 88: register BUF *bp; ! 89: ! 90: for (bp=&bufl[NBUF-1]; bp >= bufl; --bp) { ! 91: if ((bp->b_flag&BFMOD) == 0) ! 92: continue; ! 93: lock(bp->b_gate); ! 94: if ((bp->b_flag&BFMOD) != 0) ! 95: bwrite(bp, 1); ! 96: unlock(bp->b_gate); ! 97: } ! 98: } ! 99: ! 100: /* ! 101: * Synchronise all block for a particular device in the buffer cache ! 102: * and invalidate all references. ! 103: */ ! 104: bflush(dev) ! 105: register dev_t dev; ! 106: { ! 107: register BUF *bp; ! 108: ! 109: for (bp=&bufl[NBUF-1]; bp >= bufl; --bp) { ! 110: if (bp->b_dev != dev) ! 111: continue; ! 112: lock(bp->b_gate); ! 113: if (bp->b_dev == dev) { ! 114: if ((bp->b_flag&BFMOD) != 0) ! 115: bwrite(bp, 1); ! 116: bp->b_dev = NODEV; ! 117: } ! 118: unlock(bp->b_gate); ! 119: } ! 120: } ! 121: ! 122: /* ! 123: * Return a buffer containing the given block from the given device. ! 124: * If `f' is not set, the read is asynchronous and no buffer is returned. ! 125: */ ! 126: BUF * ! 127: bread(dev, bno, f) ! 128: dev_t dev; ! 129: daddr_t bno; ! 130: register int f; ! 131: { ! 132: register BUF *bp; ! 133: register int s; ! 134: ! 135: bp = bclaim(dev, bno); ! 136: if ((bp->b_flag&BFNTP) != 0) { ! 137: if (f != 0) ! 138: bp->b_flag &= ~BFASY; ! 139: else { ! 140: bp->b_flag |= BFASY; ! 141: bumap(bp); ! 142: } ! 143: bp->b_req = BREAD; ! 144: bp->b_count = BSIZE; ! 145: s = sphi(); ! 146: dblock(dev, bp); ! 147: if (f == 0) { ! 148: spl(s); ! 149: return (NULL); ! 150: } ! 151: while ((bp->b_flag&BFNTP) != 0) ! 152: sleep((char *)bp, CVBLKIO, IVBLKIO, SVBLKIO); ! 153: spl(s); ! 154: if ((bp->b_flag&BFERR) != 0) { ! 155: u.u_error = bp->b_err ? bp->b_err : EIO; ! 156: brelease(bp); ! 157: return (NULL); ! 158: } ! 159: if (bp->b_resid == BSIZE) { ! 160: brelease(bp); ! 161: return (NULL); ! 162: } ! 163: } ! 164: if (f == 0) { ! 165: brelease(bp); ! 166: return (NULL); ! 167: } ! 168: u.u_block++; ! 169: return (bp); ! 170: } ! 171: ! 172: /* ! 173: * If the requested buffer is in the buffer cache, return a pointer to ! 174: * it. If not, pick an empty buffer, set it up and return it. ! 175: */ ! 176: BUF * ! 177: bclaim(dev, bno) ! 178: dev_t dev; ! 179: daddr_t bno; ! 180: { ! 181: register BUF *bp; ! 182: register BUF *bp1; ! 183: register unsigned seqn; ! 184: register int s; ! 185: ! 186: again: ! 187: bp1 = NULL; ! 188: seqn = 0; ! 189: for (bp=&bufl[NBUF-1]; bp >= bufl; --bp) { ! 190: if (bp->b_bno == bno && bp->b_dev == dev) { ! 191: lock(bp->b_gate); ! 192: if (bp->b_bno != bno || bp->b_dev != dev) { ! 193: unlock(bp->b_gate); ! 194: goto again; ! 195: } ! 196: if ((bp->b_flag&BFERR) != 0) ! 197: bp->b_flag |= BFNTP; ! 198: bsmap(bp); ! 199: return (bp); ! 200: } ! 201: if (locked(bp->b_gate) == 0) { ! 202: if (bufseqn-bp->b_seqn >= seqn) { ! 203: bp1 = bp; ! 204: seqn = bufseqn - bp->b_seqn; ! 205: } ! 206: } ! 207: } ! 208: if (bp1 == NULL) { ! 209: s = sphi(); ! 210: for (bp=&bufl[NBUF-1]; bp >= bufl; --bp) { ! 211: if (locked(bp->b_gate) == 0) { ! 212: if (bufseqn-bp->b_seqn >= seqn) { ! 213: bp1 = bp; ! 214: seqn = bufseqn - bp->b_seqn; ! 215: } ! 216: } ! 217: } ! 218: if (bp1 == NULL) { ! 219: bufneed = 1; ! 220: sleep((char *)&bufneed, CVBLKIO, IVBLKIO, SVBLKIO); ! 221: spl(s); ! 222: goto again; ! 223: } ! 224: spl(s); ! 225: } ! 226: bp = bp1; ! 227: lock(bp->b_gate); ! 228: if ((bp->b_flag&BFMOD) != 0) { ! 229: bwrite(bp, 0); ! 230: goto again; ! 231: } ! 232: bp->b_flag = BFNTP; ! 233: bp->b_dev = dev; ! 234: bp->b_bno = bno; ! 235: bsmap(bp); ! 236: return (bp); ! 237: } ! 238: ! 239: /* ! 240: * Write the given buffer out. If `f' is set, the write is synchronous, ! 241: * otherwise asynchronous. This routine must be called with the buffer ! 242: * gate locked. ! 243: */ ! 244: bwrite(bp, f) ! 245: register BUF *bp; ! 246: { ! 247: register int s; ! 248: ! 249: if (f != 0) ! 250: bp->b_flag &= ~BFASY; ! 251: else { ! 252: bp->b_flag |= BFASY; ! 253: bumap(bp); ! 254: } ! 255: bp->b_flag |= BFNTP; ! 256: bp->b_req = BWRITE; ! 257: bp->b_count = BSIZE; ! 258: s = sphi(); ! 259: dblock(bp->b_dev, bp); ! 260: if (f == 0) { ! 261: spl(s); ! 262: return; ! 263: } ! 264: while ((bp->b_flag&BFNTP) != 0) ! 265: sleep((char *)bp, CVBLKIO, IVBLKIO, SVBLKIO); ! 266: spl(s); ! 267: } ! 268: ! 269: /* ! 270: * This is called by the driver when I/O has completed on a buffer. ! 271: */ ! 272: bdone(bp) ! 273: register BUF *bp; ! 274: { ! 275: if (bp->b_req == BWRITE) ! 276: bp->b_flag &= ~BFMOD; ! 277: if (bp->b_req == BREAD) { ! 278: if ((bp->b_flag&BFERR) != 0) ! 279: bp->b_dev = NODEV; ! 280: } ! 281: if ((bp->b_flag&BFASY) != 0) { ! 282: bp->b_flag &= ~BFASY; ! 283: brelease(bp); ! 284: } ! 285: bp->b_flag &= ~BFNTP; ! 286: wakeup((char *)bp); ! 287: } ! 288: ! 289: /* ! 290: * Release the given buffer. ! 291: */ ! 292: brelease(bp) ! 293: register BUF *bp; ! 294: { ! 295: if ((bp->b_flag&BFERR) == 0) ! 296: bp->b_seqn = bufseqn++; ! 297: else { ! 298: bp->b_flag &= ~BFERR; ! 299: bp->b_dev = NODEV; ! 300: } ! 301: bp->b_flag &= ~BFNTP; ! 302: bumap(bp); ! 303: unlock(bp->b_gate); ! 304: if (bufneed != 0) { ! 305: bufneed = 0; ! 306: wakeup((char *)&bufneed); ! 307: } ! 308: } ! 309: ! 310: /* ! 311: * Map the given buffer. ! 312: */ ! 313: bsmap(bp) ! 314: register BUF *bp; ! 315: { ! 316: bsave(bp->b_map); ! 317: bp->b_flag |= BFMAP; ! 318: bmapv(bconv(bp->b_paddr)); ! 319: } ! 320: ! 321: /* ! 322: * Unmap the given buffer. ! 323: */ ! 324: bumap(bp) ! 325: register BUF *bp; ! 326: { ! 327: if ((bp->b_flag&BFMAP) == 0) ! 328: return; ! 329: bp->b_flag &= ~BFMAP; ! 330: brest(bp->b_map); ! 331: } ! 332: ! 333: /* ! 334: * Read data from the I/O segment into kernel space. ! 335: */ ! 336: ioread(iop, v, n) ! 337: register IO *iop; ! 338: register char *v; ! 339: register unsigned n; ! 340: { ! 341: switch (iop->io_seg) { ! 342: case IOSYS: ! 343: iop->io_base += kkcopy(iop->io_base, v, n); ! 344: break; ! 345: case IOUSR: ! 346: iop->io_base += ukcopy(iop->io_base, v, n); ! 347: break; ! 348: case IOPHY: ! 349: iop->io_phys += pkcopy(iop->io_phys, v, n); ! 350: break; ! 351: } ! 352: iop->io_ioc -= n; ! 353: } ! 354: ! 355: /* ! 356: * Write data from kernel space to the I/O segment. ! 357: */ ! 358: iowrite(iop, v, n) ! 359: register IO *iop; ! 360: register char *v; ! 361: register unsigned n; ! 362: { ! 363: switch (iop->io_seg) { ! 364: case IOSYS: ! 365: iop->io_base += kkcopy(v, iop->io_base, n); ! 366: break; ! 367: case IOUSR: ! 368: iop->io_base += kucopy(v, iop->io_base, n); ! 369: break; ! 370: case IOPHY: ! 371: iop->io_phys += kpcopy(v, iop->io_phys, n); ! 372: break; ! 373: } ! 374: iop->io_ioc -= n; ! 375: } ! 376: ! 377: /* ! 378: * Get a character from the I/O segment. ! 379: */ ! 380: iogetc(iop) ! 381: register IO *iop; ! 382: { ! 383: register int c; ! 384: ! 385: if (iop->io_ioc == 0) ! 386: return (-1); ! 387: --iop->io_ioc; ! 388: if (iop->io_seg == IOSYS) ! 389: c = *iop->io_base++ & 0377; ! 390: else { ! 391: c = getubd(iop->io_base++); ! 392: if (u.u_error) ! 393: return (-1); ! 394: } ! 395: return (c); ! 396: } ! 397: ! 398: /* ! 399: * Put a character using the I/O segment. ! 400: */ ! 401: ioputc(c, iop) ! 402: register IO *iop; ! 403: { ! 404: if (iop->io_ioc == 0) ! 405: return (-1); ! 406: --iop->io_ioc; ! 407: if (iop->io_seg == IOSYS) ! 408: *iop->io_base++ = c; ! 409: else { ! 410: putubd(iop->io_base++, c); ! 411: if (u.u_error) ! 412: return (-1); ! 413: } ! 414: return (c); ! 415: } ! 416: ! 417: /* ! 418: * Given a buffer pointer, an I/O structure, a device, request type, and ! 419: * a flags word, check the I/O structure and perform the I/O request. ! 420: */ ! 421: ioreq(bp, iop, dev, req, f) ! 422: register BUF *bp; ! 423: register IO *iop; ! 424: dev_t dev; ! 425: { ! 426: register SEG *sp; ! 427: register int n; ! 428: register int s; ! 429: register CON *cp; ! 430: dold_t dold; ! 431: ! 432: if ((cp=drvmap(dev, &dold)) == NULL) ! 433: return; ! 434: lock(bp->b_gate); ! 435: n = cp->c_flag; /* n should do something with that flag */ ! 436: drest(dold); ! 437: sp = NULL; ! 438: if (iop != NULL) { ! 439: if ((f&BFBLK) != 0) { ! 440: if (blocko(iop->io_seek) != 0) { ! 441: u.u_error = EIO; ! 442: goto out; ! 443: } ! 444: } ! 445: if ((f&BFIOC) != 0) { ! 446: if ((sp=iomapvp(iop, bp)) == NULL) { ! 447: u.u_error = EIO; ! 448: goto out; ! 449: } ! 450: } ! 451: } ! 452: bp->b_flag = f|BFNTP; ! 453: bp->b_req = req; ! 454: bp->b_dev = dev; ! 455: if (iop != NULL) { ! 456: bp->b_bno = blockn(iop->io_seek); ! 457: bp->b_count = iop->io_ioc; ! 458: } ! 459: if (sp != NULL) { ! 460: bp->b_faddr = ptov( bp->b_paddr, (fsize_t) bp->b_count ); ! 461: sp->s_lrefc++; ! 462: } ! 463: s = sphi(); ! 464: dblock(dev, bp); ! 465: while ((bp->b_flag&BFNTP) != 0) ! 466: sleep((char *)bp, CVBLKIO, IVBLKIO, SVBLKIO); ! 467: spl(s); ! 468: if (sp != NULL) { ! 469: vrelse( bp->b_faddr ); ! 470: sp->s_lrefc--; ! 471: } ! 472: if (stimer.t_last != 0) ! 473: wakeup((char *)&stimer); ! 474: if ((bp->b_flag&BFERR) != 0) { ! 475: u.u_error = bp->b_err ? bp->b_err : EIO; ! 476: goto out; ! 477: } ! 478: if (iop != NULL) { ! 479: n = iop->io_ioc - bp->b_resid; ! 480: iop->io_seek += n; ! 481: iop->io_ioc -= n; ! 482: } ! 483: out: ! 484: unlock(bp->b_gate); ! 485: } ! 486: ! 487: /* ! 488: * Given an I/O structure and a buffer header, see if the addresses ! 489: * in the I/O structure are valid and set up the buffer header. ! 490: */ ! 491: SEG * ! 492: iomapvp(iop, bp) ! 493: register IO *iop; ! 494: register BUF *bp; ! 495: { ! 496: register SR *srp; ! 497: register SEG *sp; ! 498: register vaddr_t b; ! 499: ! 500: if (iop->io_seg != IOUSR) ! 501: panic("Raw I/O from non user"); ! 502: for (srp=u.u_segl; srp<&u.u_segl[NUSEG]; srp++) { ! 503: if ((sp=srp->sr_segp) == NULL) ! 504: continue; ! 505: if ((srp->sr_flag&SRFDATA) == 0) ! 506: continue; ! 507: /* Yet another bug in the 8000 C compiler ! 508: if ((long)(b=iop->io_base) < (long)srp->sr_base) ! 509: */ ! 510: if ((b=iop->io_base) < srp->sr_base) ! 511: continue; ! 512: if ((long)b+iop->io_ioc > (long)srp->sr_base + sp->s_size) ! 513: continue; ! 514: bp->b_paddr = sp->s_paddr + (vaddr_t) (b - srp->sr_base); ! 515: return (sp); ! 516: } ! 517: return (NULL); ! 518: } ! 519: ! 520: /* ! 521: * Initialise devices. ! 522: */ ! 523: devinit() ! 524: { ! 525: register DRV *dp; ! 526: register int mind; ! 527: ! 528: for ( dp = drvl, mind = 0; mind < drvn; mind++, dp++ ) { ! 529: if ((dp->d_conp != NULL) && (dp->d_conp->c_load != NULL)) { ! 530: (*dp->d_conp->c_load)(); ! 531: } ! 532: } ! 533: } ! 534: ! 535: /* ! 536: * Open a device. ! 537: */ ! 538: dopen(dev, m, f) ! 539: register dev_t dev; ! 540: { ! 541: register CON *cp; ! 542: dold_t dold; ! 543: ! 544: if ((cp=drvmap(dev, &dold)) == NULL) ! 545: return; ! 546: if ((cp->c_flag&f) == 0) { ! 547: u.u_error = ENXIO; ! 548: return; ! 549: } ! 550: (*cp->c_open)(dev, m); ! 551: drest(dold); ! 552: } ! 553: ! 554: /* ! 555: * Close a device. ! 556: */ ! 557: dclose(dev) ! 558: register dev_t dev; ! 559: { ! 560: register CON *cp; ! 561: dold_t dold; ! 562: ! 563: if ((cp=drvmap(dev, &dold)) == NULL) ! 564: return; ! 565: (*cp->c_close)(dev); ! 566: drest(dold); ! 567: } ! 568: ! 569: /* ! 570: * Call the block entry point of a device. ! 571: */ ! 572: dblock(dev, bp) ! 573: dev_t dev; ! 574: BUF *bp; ! 575: { ! 576: register CON *cp; ! 577: dold_t dold; ! 578: ! 579: if ((cp=drvmap(dev, &dold)) == NULL) ! 580: return; ! 581: (*cp->c_block)(bp); ! 582: drest(dold); ! 583: } ! 584: ! 585: /* ! 586: * Read from a device. ! 587: */ ! 588: dread(dev, iop) ! 589: register dev_t dev; ! 590: register IO *iop; ! 591: { ! 592: register CON *cp; ! 593: dold_t dold; ! 594: ! 595: if ((cp=drvmap(dev, &dold)) == NULL) ! 596: return; ! 597: (*cp->c_read)(dev, iop); ! 598: drest(dold); ! 599: } ! 600: ! 601: /* ! 602: * Write to a device. ! 603: */ ! 604: dwrite(dev, iop) ! 605: register dev_t dev; ! 606: register IO *iop; ! 607: { ! 608: register CON *cp; ! 609: dold_t dold; ! 610: ! 611: if ((cp=drvmap(dev, &dold)) == NULL) ! 612: return; ! 613: (*cp->c_write)(dev, iop); ! 614: drest(dold); ! 615: } ! 616: ! 617: /* ! 618: * Call the ioctl function for a device. ! 619: */ ! 620: dioctl(dev, com, vec) ! 621: register dev_t dev; ! 622: union ioctl *vec; ! 623: { ! 624: register CON *cp; ! 625: dold_t dold; ! 626: ! 627: if ((cp=drvmap(dev, &dold)) == NULL) ! 628: return; ! 629: (*cp->c_ioctl)(dev, com, vec); ! 630: drest(dold); ! 631: } ! 632: ! 633: /* ! 634: * Call the powerfail entry point of a device. ! 635: */ ! 636: dpower(dev) ! 637: register dev_t dev; ! 638: { ! 639: register CON *cp; ! 640: dold_t dold; ! 641: ! 642: if ((cp=drvmap(dev, &dold)) == NULL) ! 643: return; ! 644: (*cp->c_power)(dev); ! 645: drest(dold); ! 646: } ! 647: ! 648: /* ! 649: * Call the timeout entry point of a device. ! 650: */ ! 651: dtime(dev) ! 652: register dev_t dev; ! 653: { ! 654: register CON *cp; ! 655: dold_t dold; ! 656: ! 657: if ((cp=drvmap(dev, &dold)) == NULL) ! 658: return; ! 659: (*cp->c_timer)(dev); ! 660: drest(dold); ! 661: } ! 662: ! 663: /* ! 664: * Poll a device. ! 665: */ ! 666: dpoll(dev, ev, msec) ! 667: register dev_t dev; ! 668: int ev; ! 669: int msec; ! 670: { ! 671: register CON *cp; ! 672: dold_t dold; ! 673: ! 674: if ((cp=drvmap(dev, &dold)) == NULL) ! 675: return POLLNVAL; ! 676: ! 677: if ( cp->c_flag & DFPOL ) ! 678: ev = (*cp->c_poll)(dev, ev, msec); ! 679: else ! 680: ev = POLLNVAL; ! 681: ! 682: drest(dold); ! 683: return ev; ! 684: } ! 685: ! 686: /* ! 687: * Given a device, return the flags word. ! 688: */ ! 689: dflag(dev) ! 690: dev_t dev; ! 691: { ! 692: register CON *cp; ! 693: register int f; ! 694: dold_t dold; ! 695: ! 696: if ((cp=drvmap(dev, &dold)) == NULL) ! 697: return (DFERR); ! 698: f = cp->c_flag; ! 699: drest(dold); ! 700: return (f); ! 701: } ! 702: ! 703: /* ! 704: * Given a device, and a pointer to a driver map save area, save the ! 705: * current map in the driver map save area and map in the new device, ! 706: * returning a pointer to the configuration entry for that device. ! 707: */ ! 708: CON * ! 709: drvmap(dev, doldp) ! 710: dev_t dev; ! 711: dold_t *doldp; ! 712: { ! 713: register DRV *dp; ! 714: register unsigned m; ! 715: ! 716: if ((m=major(dev)) >= drvn) { ! 717: u.u_error = ENXIO; ! 718: return (NULL); ! 719: } ! 720: dp = &drvl[m]; ! 721: if (locked(dp->d_gate)) { ! 722: u.u_error = ENXIO; ! 723: return (NULL); ! 724: } ! 725: if (dp->d_conp == NULL) { ! 726: u.u_error = ENXIO; ! 727: return (NULL); ! 728: } ! 729: dsave(*doldp); ! 730: if (dp->d_map != 0) ! 731: dmapv(dp->d_map); ! 732: return (dp->d_conp); ! 733: } ! 734: ! 735: /* ! 736: * Non existant device. ! 737: */ ! 738: nonedev() ! 739: { ! 740: u.u_error = ENXIO; ! 741: } ! 742: ! 743: /* ! 744: * Null device. ! 745: */ ! 746: nulldev() ! 747: { ! 748: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.