![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vfs_vnops.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Apple XNU] / XNU / bsd / vfs|
Return to vfs_vnops.c CVS log | Up to [Apple XNU] / XNU / bsd / vfs |
1.1 root 1: /*
2: * Copyright (c) 2000 Apple Computer, Inc. All rights reserved.
3: *
4: * @APPLE_LICENSE_HEADER_START@
5: *
6: * The contents of this file constitute Original Code as defined in and
7: * are subject to the Apple Public Source License Version 1.1 (the
8: * "License"). You may not use this file except in compliance with the
9: * License. Please obtain a copy of the License at
10: * http://www.apple.com/publicsource and read it before using this file.
11: *
12: * This Original Code and all software distributed under the License are
13: * distributed on an "AS IS" basis, WITHOUT WARRANTY OF ANY KIND, EITHER
14: * EXPRESS OR IMPLIED, AND APPLE HEREBY DISCLAIMS ALL SUCH WARRANTIES,
15: * INCLUDING WITHOUT LIMITATION, ANY WARRANTIES OF MERCHANTABILITY,
16: * FITNESS FOR A PARTICULAR PURPOSE OR NON-INFRINGEMENT. Please see the
17: * License for the specific language governing rights and limitations
18: * under the License.
19: *
20: * @APPLE_LICENSE_HEADER_END@
21: */
22: /* Copyright (c) 1995 NeXT Computer, Inc. All Rights Reserved */
23: /*
24: * Copyright (c) 1982, 1986, 1989, 1993
25: * The Regents of the University of California. All rights reserved.
26: * (c) UNIX System Laboratories, Inc.
27: * All or some portions of this file are derived from material licensed
28: * to the University of California by American Telephone and Telegraph
29: * Co. or Unix System Laboratories, Inc. and are reproduced herein with
30: * the permission of UNIX System Laboratories, Inc.
31: *
32: * Redistribution and use in source and binary forms, with or without
33: * modification, are permitted provided that the following conditions
34: * are met:
35: * 1. Redistributions of source code must retain the above copyright
36: * notice, this list of conditions and the following disclaimer.
37: * 2. Redistributions in binary form must reproduce the above copyright
38: * notice, this list of conditions and the following disclaimer in the
39: * documentation and/or other materials provided with the distribution.
40: * 3. All advertising materials mentioning features or use of this software
41: * must display the following acknowledgement:
42: * This product includes software developed by the University of
43: * California, Berkeley and its contributors.
44: * 4. Neither the name of the University nor the names of its contributors
45: * may be used to endorse or promote products derived from this software
46: * without specific prior written permission.
47: *
48: * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
49: * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
50: * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
51: * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
52: * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
53: * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
54: * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
55: * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
56: * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
57: * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
58: * SUCH DAMAGE.
59: *
60: * @(#)vfs_vnops.c 8.14 (Berkeley) 6/15/95
61: *
62: * History
63: * 10-20-1997 Umesh Vaishampayan
64: * Fixed the count to be off_t rather than int.
65: */
66:
67: #include <mach_nbc.h>
68: #include <sys/param.h>
69: #include <sys/systm.h>
70: #include <sys/kernel.h>
71: #include <sys/file.h>
72: #include <sys/stat.h>
73: #include <sys/buf.h>
74: #include <sys/proc.h>
75: #include <sys/mount.h>
76: #include <sys/namei.h>
77: #include <sys/vnode.h>
78: #include <sys/ioctl.h>
79: #include <sys/tty.h>
80: #include <kern/mapfs.h>
81: #include <mach/kern_return.h>
82: #include <mach/memory_object_control.h>
83: #include <mach/vm_prot.h>
84:
85: struct fileops vnops =
86: { vn_read, vn_write, vn_ioctl, vn_select, vn_closefile };
87:
88: /*
89: * Common code for vnode open operations.
90: * Check permissions, and call the VOP_OPEN or VOP_CREATE routine.
91: */
92: vn_open(ndp, fmode, cmode)
93: register struct nameidata *ndp;
94: int fmode, cmode;
95: {
96: register struct vnode *vp;
97: register struct proc *p = ndp->ni_cnd.cn_proc;
98: register struct ucred *cred = p->p_ucred;
99: struct vattr vat;
100: struct vattr *vap = &vat;
101: int error;
102: #if MACH_NBC
103: int no_mfs = fmode & O_NO_MFS;
104: #endif /* MACH_NBC */
105:
106: if (fmode & O_CREAT) {
107: ndp->ni_cnd.cn_nameiop = CREATE;
108: ndp->ni_cnd.cn_flags = LOCKPARENT | LOCKLEAF;
109: if ((fmode & O_EXCL) == 0)
110: ndp->ni_cnd.cn_flags |= FOLLOW;
111: if (error = namei(ndp))
112: return (error);
113: if (ndp->ni_vp == NULL) {
114: VATTR_NULL(vap);
115: vap->va_type = VREG;
116: vap->va_mode = cmode;
117: if (fmode & O_EXCL)
118: vap->va_vaflags |= VA_EXCLUSIVE;
119: VOP_LEASE(ndp->ni_dvp, p, cred, LEASE_WRITE);
120: if (error = VOP_CREATE(ndp->ni_dvp, &ndp->ni_vp,
121: &ndp->ni_cnd, vap))
122: return (error);
123: fmode &= ~O_TRUNC;
124: vp = ndp->ni_vp;
125: } else {
126: VOP_ABORTOP(ndp->ni_dvp, &ndp->ni_cnd);
127: if (ndp->ni_dvp == ndp->ni_vp)
128: vrele(ndp->ni_dvp);
129: else
130: vput(ndp->ni_dvp);
131: ndp->ni_dvp = NULL;
132: vp = ndp->ni_vp;
133: if (fmode & O_EXCL) {
134: error = EEXIST;
135: goto bad;
136: }
137: fmode &= ~O_CREAT;
138: }
139: } else {
140: ndp->ni_cnd.cn_nameiop = LOOKUP;
141: ndp->ni_cnd.cn_flags = FOLLOW | LOCKLEAF;
142: if (error = namei(ndp))
143: return (error);
144: vp = ndp->ni_vp;
145: }
146: if (vp->v_type == VSOCK) {
147: error = EOPNOTSUPP;
148: goto bad;
149: }
150: if ((fmode & O_CREAT) == 0) {
151: if (fmode & FREAD && fmode & (FWRITE | O_TRUNC)) {
152: int err = 0;
153: if (vp->v_type == VDIR)
154: err = EISDIR;
155: else
156: err = vn_writechk(vp);
157: if (err && !(error = VOP_ACCESS(vp, VREAD, cred, p)))
158: error = err;
159: if (error || (error = VOP_ACCESS(vp, VREAD|VWRITE,
160: cred, p)))
161: goto bad;
162: } else if (fmode & FREAD) {
163: if ((error = VOP_ACCESS(vp, VREAD, cred, p)))
164: goto bad;
165: } else if (fmode & (FWRITE | O_TRUNC)) {
166: if (vp->v_type == VDIR) {
167: error = EISDIR;
168: goto bad;
169: }
170: if ((error = vn_writechk(vp)) ||
171: (error = VOP_ACCESS(vp, VWRITE, cred, p)))
172: goto bad;
173: }
174: }
175: if (fmode & O_TRUNC) {
176: VOP_UNLOCK(vp, 0, p); /* XXX */
177: VOP_LEASE(vp, p, cred, LEASE_WRITE);
178: vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, p); /* XXX */
179: VATTR_NULL(vap);
180: vap->va_size = 0;
181: if (error = VOP_SETATTR(vp, vap, cred, p))
182: goto bad;
183: }
184: if (error = VOP_OPEN(vp, fmode, cred, p))
185: goto bad;
186: if ((vp->v_type == VREG) && !vp->v_vm_info) // XXX WMG
187: vm_info_init(vp);
188: #if MACH_NBC
189: if ((error == 0) && (no_mfs == 0) && (vp->v_type == VREG)) {
190: /* if the file was never mapped, leave it alone */
191: if (vp->v_vm_info->pager != NULL)
192: map_vnode(vp,p);
193: }
194:
195: /*
196: * if O_NO_MFS is set on open, blow the VM cache.
197: * if someone else is using it in VM then this has no effect.
198: */
199: if ((vp->v_type == VREG) && (no_mfs != 0))
200: (void) vnode_uncache(vp);
201: #endif /* MACH_NBC */
202:
203: if (fmode & FWRITE)
204: if (++vp->v_writecount <= 0)
205: panic("vn_open: v_writecount");
206: return (0);
207: bad:
208: vput(vp);
209: return (error);
210: }
211:
212: /*
213: * Check for write permissions on the specified vnode.
214: * Prototype text segments cannot be written.
215: */
216: vn_writechk(vp)
217: register struct vnode *vp;
218: {
219:
220: /*
221: * If there's shared text associated with
222: * the vnode, try to free it up once. If
223: * we fail, we can't allow writing.
224: */
225: if ((vp->v_flag & VTEXT) && !vnode_uncache(vp))
226: return (ETXTBSY);
227: return (0);
228: }
229:
230: /*
231: * Vnode close call
232: */
233: vn_close(vp, flags, cred, p)
234: register struct vnode *vp;
235: int flags;
236: struct ucred *cred;
237: struct proc *p;
238: {
239: int error;
240: vm_map_t user_map;
241: vm_offset_t addr, addr1;
242: vm_size_t size, pageoff;
243: void * object;
244: void * pager_cport;
245: kern_return_t kret;
246:
247: #if MACH_NBC
248: if ((vp->v_type == VREG) && vp->v_vm_info)
249: unmap_vnode(vp,p);
250: #endif /* MACH_NBC */
251:
252: if (ISMAPFILEFLUSH(vp)) {
253: if (pager_cport = vnode_pager_lookup(vp,
254: vp->v_vm_info->pager)) {
255: if(object = vm_object_lookup(pager_cport)) {
256: kret = memory_object_lock_request(object,
257: 0,
258: round_page(vp->v_vm_info->vnode_size),
259: MEMORY_OBJECT_RETURN_NONE,TRUE,
260: VM_PROT_NO_CHANGE,MACH_PORT_NULL);
261: #if DIAGNOSTIC
262: if (kret != KERN_SUCCESS) {
263: printf("ubc:failed to invalidate in close\n");
264: }
265: #endif /* DIAGNOSTIC */
266: }
267: }
268: vp->v_vm_info->dirty=0;
269: }
270:
271:
272: if (flags & FWRITE)
273: vp->v_writecount--;
274: error = VOP_CLOSE(vp, flags, cred, p);
275: vrele(vp);
276: return (error);
277: }
278:
279: /*
280: * Package up an I/O request on a vnode into a uio and do it.
281: */
282: vn_rdwr(rw, vp, base, len, offset, segflg, ioflg, cred, aresid, p)
283: enum uio_rw rw;
284: struct vnode *vp;
285: caddr_t base;
286: int len;
287: off_t offset;
288: enum uio_seg segflg;
289: int ioflg;
290: struct ucred *cred;
291: int *aresid;
292: struct proc *p;
293: {
294: struct uio auio;
295: struct iovec aiov;
296: int error=0;
297:
298: /* The routines that use this under MACH_NBC, call this
299: * routine with IO_NODELOCKED on, which is incorrect.
300: * For ease of code usage in case MACH_NBC is turned off
301: * we will bypass this check and let the callers still use
302: * IO_NODELOCKED flag
303: */
304: /* FIXME XXX */
305: if ((ioflg & IO_NODELOCKED) == 0) {
306: if (rw == UIO_READ)
307: vn_lock(vp, LK_SHARED | LK_RETRY, p);
308: else
309: vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, p);
310: }
311: auio.uio_iov = &aiov;
312: auio.uio_iovcnt = 1;
313: aiov.iov_base = base;
314: aiov.iov_len = len;
315: auio.uio_resid = len;
316: auio.uio_offset = offset;
317: auio.uio_segflg = segflg;
318: auio.uio_rw = rw;
319: auio.uio_procp = p;
320: #if MACH_NBC
321: /* if the file was never mapped, leave it alone */
322: if ((vp->v_type == VREG) && (vp->v_vm_info->pager != NULL)) {
323: if (vp->v_vm_info->map_count == 0) {
324: /*
325: * file has a pager but MapFS is not involved yet
326: * get MapFS involved
327: */
328: map_vnode(vp,p); /* vn_close() will loose this map_count */
329: }
330: error = mapfs_io(vp, &auio, rw, ioflg, cred);
331: } else {
332: #endif /* MACH_NBC */
333: if (rw == UIO_READ)
334: error = VOP_READ(vp, &auio, ioflg, cred);
335: else
336: error = VOP_WRITE(vp, &auio, ioflg, cred);
337: #if MACH_NBC
338: }
339: #endif /* MACH_NBC */
340: if (aresid)
341: *aresid = auio.uio_resid;
342: else
343: if (auio.uio_resid && error == 0)
344: error = EIO;
345: if ((ioflg & IO_NODELOCKED) == 0)
346: VOP_UNLOCK(vp, 0, p);
347: return (error);
348: }
349:
350: /*
351: * File table vnode read routine.
352: */
353: vn_read(fp, uio, cred)
354: struct file *fp;
355: struct uio *uio;
356: struct ucred *cred;
357: {
358: struct vnode *vp = (struct vnode *)fp->f_data;
359: struct proc *p = uio->uio_procp;
360: int error;
361: off_t count;
362:
363: VOP_LEASE(vp, p, cred, LEASE_READ);
364: vn_lock(vp, LK_SHARED | LK_RETRY, p);
365: uio->uio_offset = fp->f_offset;
366: count = uio->uio_resid;
367:
368: if (ISMAPPEDFILE(vp)) {
369: void *object, *pager_cport;
370: kern_return_t kret;
371: struct vm_info *vmp = vp->v_vm_info;
372: if (pager_cport = vnode_pager_lookup(vp, vmp->pager)) {
373: if(object = vm_object_lookup(pager_cport)) {
374: off_t pgoffset, eoffset;
375: vm_size_t sizetoflush;
376:
377: pgoffset = trunc_page(uio->uio_offset);
378: eoffset = round_page((uio->uio_offset + uio->uio_resid));
379: sizetoflush = (vm_size_t)(eoffset - pgoffset);
380:
381: kret = memory_object_lock_request(object,
382: (vm_offset_t)pgoffset,
383: sizetoflush,
384: MEMORY_OBJECT_RETURN_DIRTY,FALSE,
385: VM_PROT_NO_CHANGE,MACH_PORT_NULL);
386: #if DIAGNOSTIC
387: if (kret != KERN_SUCCESS) {
388: printf("ubc:failed to flush before read\n");
389: }
390: #endif /* DIAGNOSTIC */
391: }
392: }
393: }
394:
395: error = VOP_READ(vp, uio, (fp->f_flag & FNONBLOCK) ? IO_NDELAY : 0, cred);
396:
397: fp->f_offset += count - uio->uio_resid;
398: VOP_UNLOCK(vp, 0, p);
399: return (error);
400: }
401:
402:
403: /*
404: * File table vnode write routine.
405: */
406: vn_write(fp, uio, cred)
407: struct file *fp;
408: struct uio *uio;
409: struct ucred *cred;
410: {
411: struct vnode *vp = (struct vnode *)fp->f_data;
412: struct proc *p = uio->uio_procp;
413: int error, ioflag = IO_UNIT;
414: off_t count;
415:
416: if (vp->v_type == VREG && (fp->f_flag & O_APPEND))
417: ioflag |= IO_APPEND;
418: if (fp->f_flag & FNONBLOCK)
419: ioflag |= IO_NDELAY;
420: if ((fp->f_flag & O_FSYNC) ||
421: (vp->v_mount && (vp->v_mount->mnt_flag & MNT_SYNCHRONOUS)))
422: ioflag |= IO_SYNC;
423: VOP_LEASE(vp, p, cred, LEASE_WRITE);
424: vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, p);
425: uio->uio_offset = fp->f_offset;
426: count = uio->uio_resid;
427:
428: if (ISMAPPEDFILE(vp)) {
429: void *object, *pager_cport;
430: kern_return_t kret;
431: struct vm_info *vmp = vp->v_vm_info;
432: if (pager_cport = vnode_pager_lookup(vp, vmp->pager)) {
433: if(object = vm_object_lookup(pager_cport)) {
434: off_t pgoffset, eoffset;
435: vm_size_t sizetoflush;
436:
437: pgoffset = trunc_page(uio->uio_offset);
438: eoffset = round_page((uio->uio_offset + uio->uio_resid));
439: sizetoflush = (vm_size_t)(eoffset - pgoffset);
440:
441: kret = memory_object_lock_request(object,
442: (vm_offset_t)pgoffset,
443: sizetoflush,
444: MEMORY_OBJECT_RETURN_DIRTY,FALSE,
445: VM_PROT_NO_CHANGE,MACH_PORT_NULL);
446: #if DIAGNOSTIC
447: if (kret != KERN_SUCCESS) {
448: printf("ubc:failed to flush before write\n");
449: }
450: #endif /* DIAGNOSTIC */
451: }
452: }
453: }
454:
455: error = VOP_WRITE(vp, uio, ioflag, cred);
456:
457: if (ISMAPPEDFILE(vp)) {
458: void *object, *pager_cport;
459: kern_return_t kret;
460: struct vm_info *vmp = vp->v_vm_info;
461: if (pager_cport = vnode_pager_lookup(vp, vmp->pager)) {
462: if(object = vm_object_lookup(pager_cport)) {
463: off_t pgoffset, eoffset;
464: vm_size_t sizetoflush;
465:
466: pgoffset = trunc_page(uio->uio_offset);
467: eoffset = round_page((uio->uio_offset + uio->uio_resid));
468: sizetoflush = (vm_size_t)(eoffset - pgoffset);
469:
470: kret = memory_object_lock_request(object,
471: (vm_offset_t)pgoffset,
472: sizetoflush,
473: MEMORY_OBJECT_RETURN_NONE,TRUE,
474: VM_PROT_NO_CHANGE,MACH_PORT_NULL);
475: #if DIAGNOSTIC
476: if (kret != KERN_SUCCESS) {
477: printf("ubc:failed to invalidate after write\n");
478: }
479: #endif /* DIAGNOSTIC */
480: }
481: }
482: vmp->dirty=1;
483: }
484:
485: if (ioflag & IO_APPEND)
486: fp->f_offset = uio->uio_offset;
487: else
488: fp->f_offset += count - uio->uio_resid;
489: VOP_UNLOCK(vp, 0, p);
490: return (error);
491: }
492:
493: /*
494: * File table vnode stat routine.
495: */
496: vn_stat(vp, sb, p)
497: struct vnode *vp;
498: register struct stat *sb;
499: struct proc *p;
500: {
501: struct vattr vattr;
502: register struct vattr *vap;
503: int error;
504: u_short mode;
505:
506: vap = &vattr;
507: error = VOP_GETATTR(vp, vap, p->p_ucred, p);
508: if (error)
509: return (error);
510: /*
511: * Copy from vattr table
512: */
513: sb->st_dev = vap->va_fsid;
514: sb->st_ino = vap->va_fileid;
515: mode = vap->va_mode;
516: switch (vp->v_type) {
517: case VREG:
518: mode |= S_IFREG;
519: break;
520: case VDIR:
521: mode |= S_IFDIR;
522: break;
523: case VBLK:
524: mode |= S_IFBLK;
525: break;
526: case VCHR:
527: mode |= S_IFCHR;
528: break;
529: case VLNK:
530: mode |= S_IFLNK;
531: break;
532: case VSOCK:
533: mode |= S_IFSOCK;
534: break;
535: case VFIFO:
536: mode |= S_IFIFO;
537: break;
538: default:
539: return (EBADF);
540: };
541: sb->st_mode = mode;
542: sb->st_nlink = vap->va_nlink;
543: sb->st_uid = vap->va_uid;
544: sb->st_gid = vap->va_gid;
545: sb->st_rdev = vap->va_rdev;
546: sb->st_size = vap->va_size;
547: sb->st_atimespec = vap->va_atime;
548: sb->st_mtimespec = vap->va_mtime;
549: sb->st_ctimespec = vap->va_ctime;
550: sb->st_blksize = vap->va_blocksize;
551: sb->st_flags = vap->va_flags;
552: /* Do not give the generation number out to unpriviledged users */
553: if (suser(p->p_ucred, &p->p_acflag))
554: sb->st_gen = 0;
555: else
556: sb->st_gen = vap->va_gen;
557: sb->st_blocks = vap->va_bytes / S_BLKSIZE;
558: return (0);
559: }
560:
561: /*
562: * File table vnode ioctl routine.
563: */
564: vn_ioctl(fp, com, data, p)
565: struct file *fp;
566: u_long com;
567: caddr_t data;
568: struct proc *p;
569: {
570: register struct vnode *vp = ((struct vnode *)fp->f_data);
571: struct vattr vattr;
572: int error;
573:
574: switch (vp->v_type) {
575:
576: case VREG:
577: case VDIR:
578: if (com == FIONREAD) {
579: if (error = VOP_GETATTR(vp, &vattr, p->p_ucred, p))
580: return (error);
581: *(int *)data = vattr.va_size - fp->f_offset;
582: return (0);
583: }
584: if (com == FIONBIO || com == FIOASYNC) /* XXX */
585: return (0); /* XXX */
586: /* fall into ... */
587:
588: default:
589: return (ENOTTY);
590:
591: case VFIFO:
592: case VCHR:
593: case VBLK:
594: error = VOP_IOCTL(vp, com, data, fp->f_flag, p->p_ucred, p);
595: if (error == 0 && com == TIOCSCTTY) {
596: if (p->p_session->s_ttyvp)
597: vrele(p->p_session->s_ttyvp);
598: p->p_session->s_ttyvp = vp;
599: VREF(vp);
600: }
601: return (error);
602: }
603: }
604:
605: /*
606: * File table vnode select routine.
607: */
608: vn_select(fp, which, p)
609: struct file *fp;
610: int which;
611: struct proc *p;
612: {
613:
614: return (VOP_SELECT(((struct vnode *)fp->f_data), which, fp->f_flag,
615: fp->f_cred, p));
616: }
617:
618: /*
619: * Check that the vnode is still valid, and if so
620: * acquire requested lock.
621: */
622: int
623: vn_lock(vp, flags, p)
624: struct vnode *vp;
625: int flags;
626: struct proc *p;
627: {
628: int error;
629:
630: do {
631: if ((flags & LK_INTERLOCK) == 0)
632: simple_lock(&vp->v_interlock);
633: if (vp->v_flag & VXLOCK) {
634: vp->v_flag |= VXWANT;
635: simple_unlock(&vp->v_interlock);
636: tsleep((caddr_t)vp, PINOD, "vn_lock", 0);
637: error = ENOENT;
638: } else {
639: error = VOP_LOCK(vp, flags | LK_INTERLOCK, p);
640: if (error == 0)
641: return (error);
642: }
643: flags &= ~LK_INTERLOCK;
644: } while (flags & LK_RETRY);
645: return (error);
646: }
647:
648: /*
649: * File table vnode close routine.
650: */
651: vn_closefile(fp, p)
652: struct file *fp;
653: struct proc *p;
654: {
655:
656: return (vn_close(((struct vnode *)fp->f_data), fp->f_flag,
657: fp->f_cred, p));
658: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.