|
|
1.1 ! root 1: /* $Header: /y/coh.386/RCS/fs1.c,v 1.4 92/07/16 16:33:32 hal Exp $ */ ! 2: /* (lgl- ! 3: * The information contained herein is a trade secret of Mark Williams ! 4: * Company, and is confidential information. It is provided under a ! 5: * license agreement, and may be copied or disclosed only under the ! 6: * terms of that agreement. Any reproduction or disclosure of this ! 7: * material without the express written authorization of Mark Williams ! 8: * Company or persuant to the license agreement is unlawful. ! 9: * ! 10: * COHERENT Version 2.3.37 ! 11: * Copyright (c) 1982, 1983, 1984. ! 12: * An unpublished work by Mark Williams Company, Chicago. ! 13: * All rights reserved. ! 14: -lgl) */ ! 15: /* ! 16: * Coherent. ! 17: * Filesystem (mostly handling of in core inodes). ! 18: * ! 19: * $Log: fs1.c,v $ ! 20: * Revision 1.4 92/07/16 16:33:32 hal ! 21: * Kernel #58 ! 22: * ! 23: * Revision 1.3 92/02/06 17:55:36 vlad ! 24: * Fix typo in ialloc panic. ! 25: * ! 26: * Revision 1.2 92/01/06 11:59:17 hal ! 27: * Compile with cc.mwc. ! 28: * ! 29: * Revision 1.1 88/03/24 16:13:47 src ! 30: * Initial revision ! 31: * ! 32: * 86/12/13 Allan Cornish /usr/src/sys/coh/fs1.c ! 33: * isync() no longer updates the disk image of a character device inode. ! 34: * ! 35: * 86/11/19 Allan Cornish /usr/src/sys/coh/fs1.c ! 36: * idirent() initializes the (new) (IO).io_flag field to 0. ! 37: */ ! 38: #include <sys/coherent.h> ! 39: #include <sys/buf.h> ! 40: #include <canon.h> ! 41: #include <sys/dir.h> ! 42: #include <errno.h> ! 43: #include <sys/filsys.h> ! 44: #include <sys/ino.h> ! 45: #include <sys/inode.h> ! 46: #include <sys/io.h> ! 47: #include <sys/mount.h>> ! 48: #include <sys/stat.h> ! 49: ! 50: /* ! 51: * Get character for `ftoi' depending on what space the characters are ! 52: * coming from. ! 53: */ ! 54: #define ftoic(p) (u.u_io.io_seg==IOSYS ? *p : getubd(p)) ! 55: ! 56: /* ! 57: * Map the given filename to an inode. If an error is encountered, ! 58: * `u.u_error' is set. `u.u_error' is always returned. As this routine ! 59: * needs to set several things, depending on the type of access, `t', ! 60: * there are places in the processes' user area reserved for this routine ! 61: * to set. These are defined in the user process structure. The seek ! 62: * position is always set to the position of the directory entry of the ! 63: * child if the child exists or the first free position if it doesn't. ! 64: * 'r' => Reference. A pointer to the child's inode is returned locked. ! 65: * 'c' => Create. If the child exists, a pointer to the inode is returned ! 66: * locked. Otherwise if the parent directory exists, a pointer to ! 67: * the parent directory is returned locked. Otherwise, an error. ! 68: * 'u' => Unlink. The parent directory is returned unlocked. The child's ! 69: * inode number is returned. The seek position is also set. ! 70: */ ! 71: ftoi(np, t) ! 72: char *np; ! 73: { ! 74: register INODE *cip; ! 75: register char *cp; ! 76: register int c; ! 77: register struct direct *dp; ! 78: register BUF *bp; ! 79: off_t cseek, fseek, s; ! 80: int fflag, mflag; ! 81: dev_t dev; ! 82: ino_t ino; ! 83: daddr_t b; ! 84: ! 85: u.u_cdirn = 0; ! 86: u.u_cdiri = NULL; ! 87: u.u_pdiri = NULL; ! 88: if ((c=ftoic(np++)) != '/') ! 89: cip = u.u_cdir; ! 90: else { ! 91: c = ftoic(np++); ! 92: cip = u.u_rdir; ! 93: } ! 94: while (c == '/') ! 95: c = ftoic(np++); ! 96: ilock(cip); ! 97: cip->i_refc++; ! 98: if (c == '\0') { ! 99: if (t == 'r') { ! 100: u.u_cdiri = cip; ! 101: return (u.u_error); ! 102: } ! 103: u.u_error = ENOENT; ! 104: idetach(cip); ! 105: return (u.u_error); ! 106: } ! 107: for (;;) { ! 108: cp = u.u_direct.d_name; ! 109: while (c!='/' && c!='\0') { ! 110: if (cp < &u.u_direct.d_name[DIRSIZ]) ! 111: *cp++ = c; ! 112: c = ftoic(np++); ! 113: } ! 114: while (c == '/') ! 115: c = ftoic(np++); ! 116: while (cp < &u.u_direct.d_name[DIRSIZ]) ! 117: *cp++ = '\0'; ! 118: if ((cip->i_mode&IFMT) != IFDIR) ! 119: u.u_error = ENOTDIR; ! 120: else ! 121: iaccess(cip, IPE); ! 122: if (u.u_error) { ! 123: idetach(cip); ! 124: return (u.u_error); ! 125: } ! 126: cp = u.u_direct.d_name; ! 127: if (cip->i_ino==ROOTIN && cip->i_dev!=rootdev) ! 128: if (*cp++=='.' && *cp++=='.' && *cp++=='\0') ! 129: cip = ftoim(cip); ! 130: b = 0; ! 131: fflag = 0; ! 132: mflag = 0; ! 133: cseek = 0; ! 134: s = cip->i_size; ! 135: while (s > 0) { ! 136: if ((bp=vread(cip, b++)) == NULL) { ! 137: idetach(cip); ! 138: return (u.u_error); ! 139: } ! 140: dp = bp->b_vaddr; ! 141: while (dp < bp->b_vaddr+BSIZE) { ! 142: if ((s-=sizeof(*dp)) < 0) ! 143: break; ! 144: if ((ino=dp->d_ino) == 0) { ! 145: if (fflag == 0) { ! 146: fflag++; ! 147: fseek = cseek; ! 148: } ! 149: } else { ! 150: if (direq(dp)) { ! 151: canino(ino); ! 152: mflag = 1; ! 153: s = 0; ! 154: break; ! 155: } ! 156: } ! 157: cseek += sizeof(*dp); ! 158: dp++; ! 159: } ! 160: brelease(bp); ! 161: } ! 162: dev = cip->i_dev; ! 163: if (fflag == 0) ! 164: fseek = cseek; ! 165: if (mflag == 0) { ! 166: if (c=='\0' && t=='c') { ! 167: u.u_pdiri = cip; ! 168: u.u_io.io_seek = fseek; ! 169: } else { ! 170: u.u_error = ENOENT; ! 171: idetach(cip); ! 172: } ! 173: return (u.u_error); ! 174: } ! 175: if (c == '\0') { ! 176: if (t == 'u') { ! 177: u.u_cdirn = ino; ! 178: u.u_pdiri = cip; ! 179: u.u_io.io_seek = cseek; ! 180: return (u.u_error); ! 181: } ! 182: idetach(cip); ! 183: u.u_cdiri = iattach(dev, ino); ! 184: return (u.u_error); ! 185: } ! 186: idetach(cip); ! 187: if ((cip=iattach(dev, ino)) == NULL) ! 188: return (u.u_error); ! 189: } ! 190: } ! 191: ! 192: /* ! 193: * Given an inode which is the root of a file system, return the inode ! 194: * on which the file system was mounted. ! 195: */ ! 196: INODE * ! 197: ftoim(ip) ! 198: register INODE *ip; ! 199: { ! 200: register MOUNT *mp; ! 201: ! 202: for (mp=mountp; mp!=NULL; mp=mp->m_next) { ! 203: if (mp->m_dev == ip->i_dev) { ! 204: idetach(ip); ! 205: ip = mp->m_ip; ! 206: ilock(ip); ! 207: ip->i_refc++; ! 208: break; ! 209: } ! 210: } ! 211: return (ip); ! 212: } ! 213: ! 214: /* ! 215: * Compare the string in `u.u_direct.d_name' with the name in the ! 216: * given directory pointer. ! 217: */ ! 218: direq(dp) ! 219: struct direct *dp; ! 220: { ! 221: register char *cp1, *cp2; ! 222: register unsigned n; ! 223: ! 224: if (dp->d_ino == 0) ! 225: return (0); ! 226: cp1 = dp->d_name; ! 227: cp2 = u.u_direct.d_name; ! 228: n = DIRSIZ; ! 229: do { ! 230: if (*cp1++ != *cp2++) ! 231: return (0); ! 232: } while (--n); ! 233: return (1); ! 234: } ! 235: ! 236: /* ! 237: * Make an inode of the given mode and device. The parent directory, ! 238: * name and such stuff is set by ftoi. ! 239: */ ! 240: INODE * ! 241: imake(mode, rdev) ! 242: unsigned mode; ! 243: dev_t rdev; ! 244: { ! 245: register INODE *ip; ! 246: ! 247: ip = NULL; ! 248: mode &= ~u.u_umask; ! 249: if ((mode&ISVTXT)!=0 && super()==0) ! 250: goto det; ! 251: if (iaccess(u.u_pdiri, IPW) == 0) ! 252: goto det; ! 253: if ((ip=ialloc(u.u_pdiri->i_dev, mode)) == NULL) ! 254: goto det; ! 255: ip->i_nlink = 1; ! 256: ip->i_a.i_rdev = rdev; ! 257: idirent(ip->i_ino); ! 258: iamc(ip); /* creat/mknod - atime/mtime/ctime */ ! 259: det: ! 260: idetach(u.u_pdiri); ! 261: return (ip); ! 262: } ! 263: ! 264: /* ! 265: * Write a directory entry out. Everything necessary has been conveniently ! 266: * set by `ftoi', except the new inode number of this directory entry. ! 267: */ ! 268: idirent(ino) ! 269: { ! 270: u.u_direct.d_ino = ino; ! 271: canino(u.u_direct.d_ino); ! 272: u.u_io.io_ioc = sizeof (struct direct); ! 273: u.u_io.io.vbase = &u.u_direct; ! 274: u.u_io.io_seg = IOSYS; ! 275: u.u_io.io_flag = 0; ! 276: iwrite(u.u_pdiri, &u.u_io); ! 277: } ! 278: ! 279: /* ! 280: * Return a pointer to a locked inode in core containing the given ! 281: * inode number and device. ! 282: */ ! 283: INODE * ! 284: iattach(dev, ino) ! 285: { ! 286: register INODE *ip; ! 287: register INODE *fip; ! 288: register unsigned lrt; ! 289: register MOUNT *mp; ! 290: ! 291: for (;;) { ! 292: fip = NULL; ! 293: for (ip=&inodep[NINODE-1]; ip>=inodep; --ip) { ! 294: if (ip->i_ino==ino && ip->i_dev==dev) ! 295: break; ! 296: if (ip->i_refc == 0) { ! 297: if (fip==NULL || ip->i_lrt<lrt) { ! 298: fip = ip; ! 299: lrt = ip->i_lrt; ! 300: } ! 301: } ! 302: } ! 303: if (ip < inodep) { ! 304: if ((ip=fip) == NULL) { ! 305: devmsg(dev, "Inode table overflow"); ! 306: u.u_error = ENFILE; ! 307: return (NULL); ! 308: } ! 309: ilock(ip); ! 310: if (ip->i_refc != 0) { ! 311: iunlock(ip); ! 312: continue; ! 313: } ! 314: ip->i_dev = dev; ! 315: ip->i_ino = ino; ! 316: ip->i_refc = 1; ! 317: ip->i_lrt = timer.t_time; ! 318: if (icopydm(ip) == 0) { ! 319: ip->i_ino = 0; ! 320: ip->i_refc = 0; ! 321: iunlock(ip); ! 322: return (NULL); ! 323: } ! 324: return (ip); ! 325: } ! 326: if ((ip->i_flag&IFMNT) != 0) { ! 327: for (mp=mountp; mp!=NULL; mp=mp->m_next) { ! 328: if (mp->m_ip == ip) { ! 329: ino = ROOTIN; ! 330: dev = mp->m_dev; ! 331: break; ! 332: } ! 333: } ! 334: continue; ! 335: } ! 336: ilock(ip); ! 337: if (ip->i_ino!=ino || ip->i_dev!=dev) { ! 338: iunlock(ip); ! 339: continue; ! 340: } ! 341: if (ip->i_refc < 0) ! 342: panic("ialloc(%x)", ip); ! 343: ip->i_refc++; ! 344: ip->i_lrt = timer.t_time; ! 345: return (ip); ! 346: } ! 347: } ! 348: ! 349: /* ! 350: * Given a locked inode, deaccess it. ! 351: */ ! 352: idetach(ip) ! 353: register INODE *ip; ! 354: { ! 355: if (ilocked(ip)==0 || ip->i_refc<=0) ! 356: panic("idetach(%p)", ip); ! 357: if (--ip->i_refc == 0) { ! 358: #if 1 ! 359: if (ip->i_rl != NULL) ! 360: panic("idetach(%p) with locked records", ip); ! 361: #endif ! 362: if ((ip->i_flag&(IFACC|IFMOD|IFCRT)) != 0 ! 363: || ip->i_nlink == 0) ! 364: icopymd(ip); ! 365: } ! 366: iunlock(ip); ! 367: } ! 368: ! 369: /* ! 370: * Given a inode which isn't locked, lock it and then deaccess. ! 371: */ ! 372: ldetach(ip) ! 373: register INODE *ip; ! 374: { ! 375: ilock(ip); ! 376: idetach(ip); ! 377: } ! 378: ! 379: /* ! 380: * A specialized routine for finding whether the given inode may be unlinked. ! 381: * Quite simple you say, but we already have an inode locked and could run ! 382: * into gating problems if we were to lock another. So we look through the ! 383: * cache to see if the inode is there. If it is, we can easily tell. If it ! 384: * isn't, `icopydm' is called with a static. This routine is only used by ! 385: * `uunlink'. ! 386: */ ! 387: iucheck(dev, ino) ! 388: register dev_t dev; ! 389: register ino_t ino; ! 390: { ! 391: register INODE *ip; ! 392: INODE inode; ! 393: ! 394: for (ip=&inodep[NINODE-1]; ip>=inodep; --ip) { ! 395: if (ip->i_ino==ino && ip->i_dev==dev) ! 396: break; ! 397: } ! 398: if (ip < inodep) { ! 399: ip = &inode; ! 400: ip->i_dev = dev; ! 401: ip->i_ino = ino; ! 402: if (icopydm(ip) == 0) ! 403: return (0); ! 404: } ! 405: if ((ip->i_mode&IFMT) == IFDIR) { ! 406: if (super() == 0) ! 407: return (0); ! 408: } ! 409: return (1); ! 410: } ! 411: ! 412: /* ! 413: * Copy an inode from disk to memory performing canonization. ! 414: */ ! 415: icopydm(ip) ! 416: register INODE *ip; ! 417: { ! 418: register struct dinode *dip; ! 419: register BUF *bp; ! 420: register ino_t ino; ! 421: struct dinode dinode; ! 422: vaddr_t v; ! 423: ! 424: ip->i_flag = 0; ! 425: ino = ip->i_ino; ! 426: ! 427: if ((bp=bread(ip->i_dev, (daddr_t)iblockn(ino), 1)) == NULL) ! 428: return (0); ! 429: ! 430: dip = &dinode; ! 431: v = (char *)((struct dinode *)bp->b_vaddr + iblocko(ino)); ! 432: kkcopy( v, dip, sizeof(dinode)); ! 433: brelease(bp); ! 434: ip->i_mode = dip->di_mode; ! 435: canshort(ip->i_mode); ! 436: ip->i_nlink = dip->di_nlink; ! 437: canshort(ip->i_nlink); ! 438: ip->i_uid = dip->di_uid; ! 439: canshort(ip->i_uid); ! 440: ip->i_gid = dip->di_gid; ! 441: canshort(ip->i_gid); ! 442: ip->i_size = dip->di_size; ! 443: cansize(ip->i_size); ! 444: ! 445: switch (ip->i_mode&IFMT) { ! 446: case IFBLK: ! 447: case IFCHR: ! 448: ip->i_a.i_rdev = dip->di_a.di_rdev; ! 449: candev(ip->i_a.i_rdev); ! 450: break; ! 451: case IFREG: ! 452: case IFDIR: ! 453: l3tol(ip->i_a.i_addr, dip->di_a.di_addb, NADDR); ! 454: break; ! 455: case IFPIPE: ! 456: l3tol(ip->i_pipe, dip->di_addp, ND); ! 457: ip->i_pnc = dip->di_pnc; ! 458: canint(ip->i_pnc); ! 459: ip->i_prx = dip->di_prx; ! 460: canint(ip->i_prx); ! 461: ip->i_pwx = dip->di_pwx; ! 462: canint(ip->i_pwx); ! 463: break; ! 464: default: ! 465: kclear(&ip->i_a, sizeof(ip->i_a)); ! 466: break; ! 467: } ! 468: ! 469: ip->i_atime = dip->di_atime; ! 470: cantime(ip->i_atime); ! 471: ip->i_mtime = dip->di_mtime; ! 472: cantime(ip->i_mtime); ! 473: ip->i_ctime = dip->di_ctime; ! 474: cantime(ip->i_ctime); ! 475: ip->i_rl = NULL; ! 476: return (1); ! 477: } ! 478: ! 479: /* ! 480: * Copy an inode from memory back on to disk performing canonization. ! 481: */ ! 482: icopymd(ip) ! 483: register INODE *ip; ! 484: { ! 485: register struct dinode *dip; ! 486: register BUF *bp; ! 487: register ino_t ino; ! 488: struct dinode dinode; ! 489: vaddr_t v; ! 490: ! 491: if (getment(ip->i_dev, 0) == NULL) ! 492: return; ! 493: ! 494: ino = ip->i_ino; ! 495: if (ip->i_refc==0 && ip->i_nlink==0 && ino!=BADFIN && ino!=ROOTIN) { ! 496: iclear(ip); ! 497: ip->i_lrt = 0; ! 498: ip->i_mode = 0; ! 499: ifree(ip->i_dev, ino); ! 500: } ! 501: ! 502: dip = &dinode; ! 503: dip->di_mode = ip->i_mode; ! 504: canshort(dip->di_mode); ! 505: dip->di_nlink = ip->i_nlink; ! 506: canshort(dip->di_nlink); ! 507: dip->di_uid = ip->i_uid; ! 508: canshort(dip->di_uid); ! 509: dip->di_gid = ip->i_gid; ! 510: canshort(dip->di_gid); ! 511: dip->di_size = ip->i_size; ! 512: cansize(dip->di_size); ! 513: ! 514: switch (ip->i_mode&IFMT) { ! 515: case IFBLK: ! 516: case IFCHR: ! 517: dip->di_a.di_rdev = ip->i_a.i_rdev; ! 518: candev(dip->di_a.di_rdev); ! 519: break; ! 520: case IFREG: ! 521: case IFDIR: ! 522: ltol3(dip->di_addr, ip->i_a.i_addr, NADDR); ! 523: break; ! 524: case IFPIPE: ! 525: ltol3(dip->di_addp, ip->i_pipe, ND); ! 526: dip->di_pnc = ip->i_pnc; ! 527: canshort(dip->di_pnc); ! 528: dip->di_prx = ip->i_prx; ! 529: canshort(dip->di_prx); ! 530: dip->di_pwx = ip->i_pwx; ! 531: canshort(dip->di_pwx); ! 532: break; ! 533: default: ! 534: kclear(&dip->di_a, sizeof(dip->di_a)); ! 535: break; ! 536: } ! 537: ! 538: dip->di_atime = ip->i_atime; ! 539: cantime(dip->di_atime); ! 540: dip->di_mtime = ip->i_mtime; ! 541: cantime(dip->di_mtime); ! 542: dip->di_ctime = ip->i_ctime; ! 543: cantime(dip->di_ctime); ! 544: ! 545: if ((bp=bread(ip->i_dev, (daddr_t)iblockn(ino), 1)) == NULL) ! 546: return; ! 547: ! 548: v = (char *)((struct dinode *)bp->b_vaddr + iblocko(ino)); ! 549: kkcopy(dip, v, sizeof(dinode)); ! 550: bp->b_flag |= BFMOD; ! 551: brelease(bp); ! 552: ip->i_flag &= ~(IFACC|IFMOD|IFCRT); ! 553: } ! 554: ! 555: /* ! 556: * Copy all relevant inodes out on device `dev'. ! 557: */ ! 558: isync(dev) ! 559: register dev_t dev; ! 560: { ! 561: register INODE *ip; ! 562: ! 563: for (ip=&inodep[NINODE-1]; ip>=inodep; --ip) { ! 564: if (ip->i_refc == 0) ! 565: continue; ! 566: if (ip->i_dev != dev) ! 567: continue; ! 568: if ( (ip->i_mode & IFMT) == IFCHR ) ! 569: continue; ! 570: if ((ip->i_flag&(IFACC|IFMOD|IFCRT)) == 0) ! 571: continue; ! 572: icopymd(ip); ! 573: } ! 574: } ! 575: ! 576: /* ! 577: * Clear the given inode and all space associated with it. ! 578: */ ! 579: iclear(ip) ! 580: register INODE *ip; ! 581: { ! 582: register int n; ! 583: register daddr_t b; ! 584: ! 585: switch (ip->i_mode&IFMT) { ! 586: case IFPIPE: ! 587: ip->i_pnc = 0; ! 588: ip->i_prx = 0; ! 589: ip->i_pwx = 0; ! 590: n = ND; ! 591: break; ! 592: case IFDIR: ! 593: case IFREG: ! 594: n = NADDR; ! 595: break; ! 596: default: ! 597: return; ! 598: } ! 599: while (n > ND) { ! 600: if ((b=ip->i_a.i_addr[--n]) != 0) ! 601: indfree(ip->i_dev, b, 1+n-ND); ! 602: } ! 603: while (n > 0) { ! 604: if ((b=ip->i_a.i_addr[--n]) != 0) ! 605: bfree(ip->i_dev, b); ! 606: } ! 607: ip->i_size = 0; ! 608: kclear(ip->i_a.i_addr, sizeof(ip->i_a.i_addr)); ! 609: iamc(ip); /* creat/pipe - atime/mtime/ctime */ ! 610: } ! 611: ! 612: /* ! 613: * Copy the appropriate information from the inode to the stat buffer. ! 614: */ ! 615: istat(ip, sbp) ! 616: register INODE *ip; ! 617: register struct stat *sbp; ! 618: { ! 619: sbp->st_dev = ip->i_dev; ! 620: sbp->st_ino = ip->i_ino; ! 621: sbp->st_mode = ip->i_mode; ! 622: sbp->st_nlink = ip->i_nlink; ! 623: sbp->st_uid = ip->i_uid; ! 624: sbp->st_gid = ip->i_gid; ! 625: sbp->st_rdev = NODEV; ! 626: sbp->st_size = ip->i_size; ! 627: sbp->st_atime = ip->i_atime; ! 628: sbp->st_mtime = ip->i_mtime; ! 629: sbp->st_ctime = ip->i_ctime; ! 630: switch (ip->i_mode&IFMT) { ! 631: case IFBLK: ! 632: case IFCHR: ! 633: sbp->st_rdev = ip->i_a.i_rdev; ! 634: sbp->st_size = 0; ! 635: break; ! 636: case IFPIPE: ! 637: sbp->st_size = ip->i_pnc; ! 638: break; ! 639: } ! 640: } ! 641: ! 642: /* ! 643: * See if it is possible to access the given inode with the bits in ! 644: * the given mode. ! 645: * If the mode includes writing, and i_refc is > 1, then check for ! 646: * shared text problems. ! 647: */ ! 648: iaccess(ip, mode) ! 649: register INODE *ip; ! 650: register int mode; ! 651: { ! 652: if ((imode(ip, u.u_uid, u.u_gid)&mode) != mode) { ! 653: u.u_error = EACCES; ! 654: return (0); ! 655: } ! 656: if ((mode&IPW) != 0 && ip->i_refc > 1 && sbusy(ip)) { ! 657: u.u_error = ETXTBSY; ! 658: return (0); ! 659: } ! 660: return (1); ! 661: } ! 662: ! 663: /* ! 664: * Get the maximum allowable mode on a file. ! 665: */ ! 666: imode(ip, uid, gid) ! 667: register INODE *ip; ! 668: { ! 669: if (uid == 0) ! 670: return (IPR|IPW|IPE); ! 671: if (uid == ip->i_uid) ! 672: return ((ip->i_mode>>6)&07); ! 673: if (gid == ip->i_gid) ! 674: return ((ip->i_mode>>3)&07); ! 675: return (ip->i_mode&07); ! 676: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.