![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to vpc.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Qemu by Fabrice Bellard] / qemu / block|
Return to vpc.c CVS log | Up to [Qemu by Fabrice Bellard] / qemu / block |
1.1 root 1: /*
1.1.1.2 root 2: * Block driver for Connectix / Microsoft Virtual PC images
1.1 root 3: *
4: * Copyright (c) 2005 Alex Beregszaszi
5: * Copyright (c) 2009 Kevin Wolf <[email protected]>
6: *
7: * Permission is hereby granted, free of charge, to any person obtaining a copy
8: * of this software and associated documentation files (the "Software"), to deal
9: * in the Software without restriction, including without limitation the rights
10: * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11: * copies of the Software, and to permit persons to whom the Software is
12: * furnished to do so, subject to the following conditions:
13: *
14: * The above copyright notice and this permission notice shall be included in
15: * all copies or substantial portions of the Software.
16: *
17: * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18: * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19: * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
20: * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21: * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22: * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
23: * THE SOFTWARE.
24: */
25: #include "qemu-common.h"
26: #include "block_int.h"
27: #include "module.h"
28:
29: /**************************************************************/
30:
31: #define HEADER_SIZE 512
32:
33: //#define CACHE
34:
35: enum vhd_type {
36: VHD_FIXED = 2,
37: VHD_DYNAMIC = 3,
38: VHD_DIFFERENCING = 4,
39: };
40:
41: // Seconds since Jan 1, 2000 0:00:00 (UTC)
42: #define VHD_TIMESTAMP_BASE 946684800
43:
44: // always big-endian
45: struct vhd_footer {
46: char creator[8]; // "conectix"
47: uint32_t features;
48: uint32_t version;
49:
50: // Offset of next header structure, 0xFFFFFFFF if none
51: uint64_t data_offset;
52:
53: // Seconds since Jan 1, 2000 0:00:00 (UTC)
54: uint32_t timestamp;
55:
56: char creator_app[4]; // "vpc "
57: uint16_t major;
58: uint16_t minor;
59: char creator_os[4]; // "Wi2k"
60:
61: uint64_t orig_size;
62: uint64_t size;
63:
64: uint16_t cyls;
65: uint8_t heads;
66: uint8_t secs_per_cyl;
67:
68: uint32_t type;
69:
70: // Checksum of the Hard Disk Footer ("one's complement of the sum of all
71: // the bytes in the footer without the checksum field")
72: uint32_t checksum;
73:
74: // UUID used to identify a parent hard disk (backing file)
75: uint8_t uuid[16];
76:
77: uint8_t in_saved_state;
78: };
79:
80: struct vhd_dyndisk_header {
81: char magic[8]; // "cxsparse"
82:
83: // Offset of next header structure, 0xFFFFFFFF if none
84: uint64_t data_offset;
85:
86: // Offset of the Block Allocation Table (BAT)
87: uint64_t table_offset;
88:
89: uint32_t version;
90: uint32_t max_table_entries; // 32bit/entry
91:
92: // 2 MB by default, must be a power of two
93: uint32_t block_size;
94:
95: uint32_t checksum;
96: uint8_t parent_uuid[16];
97: uint32_t parent_timestamp;
98: uint32_t reserved;
99:
100: // Backing file name (in UTF-16)
101: uint8_t parent_name[512];
102:
103: struct {
104: uint32_t platform;
105: uint32_t data_space;
106: uint32_t data_length;
107: uint32_t reserved;
108: uint64_t data_offset;
109: } parent_locator[8];
110: };
111:
112: typedef struct BDRVVPCState {
113: BlockDriverState *hd;
114:
115: uint8_t footer_buf[HEADER_SIZE];
116: uint64_t free_data_block_offset;
117: int max_table_entries;
118: uint32_t *pagetable;
119: uint64_t bat_offset;
120: uint64_t last_bitmap_offset;
121:
122: uint32_t block_size;
123: uint32_t bitmap_size;
124:
125: #ifdef CACHE
126: uint8_t *pageentry_u8;
127: uint32_t *pageentry_u32;
128: uint16_t *pageentry_u16;
129:
130: uint64_t last_bitmap;
131: #endif
132: } BDRVVPCState;
133:
134: static uint32_t vpc_checksum(uint8_t* buf, size_t size)
135: {
136: uint32_t res = 0;
137: int i;
138:
139: for (i = 0; i < size; i++)
140: res += buf[i];
141:
142: return ~res;
143: }
144:
145:
146: static int vpc_probe(const uint8_t *buf, int buf_size, const char *filename)
147: {
148: if (buf_size >= 8 && !strncmp((char *)buf, "conectix", 8))
149: return 100;
150: return 0;
151: }
152:
1.1.1.4 ! root 153: static int vpc_open(BlockDriverState *bs, int flags)
1.1 root 154: {
155: BDRVVPCState *s = bs->opaque;
1.1.1.4 ! root 156: int i;
1.1 root 157: struct vhd_footer* footer;
158: struct vhd_dyndisk_header* dyndisk_header;
159: uint8_t buf[HEADER_SIZE];
160: uint32_t checksum;
161:
1.1.1.4 ! root 162: if (bdrv_pread(bs->file, 0, s->footer_buf, HEADER_SIZE) != HEADER_SIZE)
1.1 root 163: goto fail;
164:
165: footer = (struct vhd_footer*) s->footer_buf;
166: if (strncmp(footer->creator, "conectix", 8))
167: goto fail;
168:
169: checksum = be32_to_cpu(footer->checksum);
170: footer->checksum = 0;
171: if (vpc_checksum(s->footer_buf, HEADER_SIZE) != checksum)
172: fprintf(stderr, "block-vpc: The header checksum of '%s' is "
1.1.1.4 ! root 173: "incorrect.\n", bs->filename);
1.1 root 174:
175: // The visible size of a image in Virtual PC depends on the geometry
176: // rather than on the size stored in the footer (the size in the footer
177: // is too large usually)
178: bs->total_sectors = (int64_t)
179: be16_to_cpu(footer->cyls) * footer->heads * footer->secs_per_cyl;
180:
1.1.1.4 ! root 181: if (bdrv_pread(bs->file, be64_to_cpu(footer->data_offset), buf, HEADER_SIZE)
1.1 root 182: != HEADER_SIZE)
183: goto fail;
184:
185: dyndisk_header = (struct vhd_dyndisk_header*) buf;
186:
187: if (strncmp(dyndisk_header->magic, "cxsparse", 8))
188: goto fail;
189:
190:
191: s->block_size = be32_to_cpu(dyndisk_header->block_size);
192: s->bitmap_size = ((s->block_size / (8 * 512)) + 511) & ~511;
193:
194: s->max_table_entries = be32_to_cpu(dyndisk_header->max_table_entries);
195: s->pagetable = qemu_malloc(s->max_table_entries * 4);
196:
197: s->bat_offset = be64_to_cpu(dyndisk_header->table_offset);
1.1.1.4 ! root 198: if (bdrv_pread(bs->file, s->bat_offset, s->pagetable,
1.1 root 199: s->max_table_entries * 4) != s->max_table_entries * 4)
200: goto fail;
201:
202: s->free_data_block_offset =
203: (s->bat_offset + (s->max_table_entries * 4) + 511) & ~511;
204:
205: for (i = 0; i < s->max_table_entries; i++) {
206: be32_to_cpus(&s->pagetable[i]);
207: if (s->pagetable[i] != 0xFFFFFFFF) {
208: int64_t next = (512 * (int64_t) s->pagetable[i]) +
209: s->bitmap_size + s->block_size;
210:
211: if (next> s->free_data_block_offset)
212: s->free_data_block_offset = next;
213: }
214: }
215:
216: s->last_bitmap_offset = (int64_t) -1;
217:
218: #ifdef CACHE
219: s->pageentry_u8 = qemu_malloc(512);
220: s->pageentry_u32 = s->pageentry_u8;
221: s->pageentry_u16 = s->pageentry_u8;
222: s->last_pagetable = -1;
223: #endif
224:
225: return 0;
226: fail:
227: return -1;
228: }
229:
230: /*
231: * Returns the absolute byte offset of the given sector in the image file.
232: * If the sector is not allocated, -1 is returned instead.
233: *
234: * The parameter write must be 1 if the offset will be used for a write
235: * operation (the block bitmaps is updated then), 0 otherwise.
236: */
237: static inline int64_t get_sector_offset(BlockDriverState *bs,
238: int64_t sector_num, int write)
239: {
240: BDRVVPCState *s = bs->opaque;
241: uint64_t offset = sector_num * 512;
242: uint64_t bitmap_offset, block_offset;
243: uint32_t pagetable_index, pageentry_index;
244:
245: pagetable_index = offset / s->block_size;
246: pageentry_index = (offset % s->block_size) / 512;
247:
248: if (pagetable_index >= s->max_table_entries || s->pagetable[pagetable_index] == 0xffffffff)
249: return -1; // not allocated
250:
251: bitmap_offset = 512 * (uint64_t) s->pagetable[pagetable_index];
252: block_offset = bitmap_offset + s->bitmap_size + (512 * pageentry_index);
253:
254: // We must ensure that we don't write to any sectors which are marked as
255: // unused in the bitmap. We get away with setting all bits in the block
256: // bitmap each time we write to a new block. This might cause Virtual PC to
257: // miss sparse read optimization, but it's not a problem in terms of
258: // correctness.
259: if (write && (s->last_bitmap_offset != bitmap_offset)) {
260: uint8_t bitmap[s->bitmap_size];
261:
262: s->last_bitmap_offset = bitmap_offset;
263: memset(bitmap, 0xff, s->bitmap_size);
1.1.1.4 ! root 264: bdrv_pwrite_sync(bs->file, bitmap_offset, bitmap, s->bitmap_size);
1.1 root 265: }
266:
267: // printf("sector: %" PRIx64 ", index: %x, offset: %x, bioff: %" PRIx64 ", bloff: %" PRIx64 "\n",
268: // sector_num, pagetable_index, pageentry_index,
269: // bitmap_offset, block_offset);
270:
271: // disabled by reason
272: #if 0
273: #ifdef CACHE
274: if (bitmap_offset != s->last_bitmap)
275: {
276: lseek(s->fd, bitmap_offset, SEEK_SET);
277:
278: s->last_bitmap = bitmap_offset;
279:
280: // Scary! Bitmap is stored as big endian 32bit entries,
281: // while we used to look it up byte by byte
282: read(s->fd, s->pageentry_u8, 512);
283: for (i = 0; i < 128; i++)
284: be32_to_cpus(&s->pageentry_u32[i]);
285: }
286:
287: if ((s->pageentry_u8[pageentry_index / 8] >> (pageentry_index % 8)) & 1)
288: return -1;
289: #else
290: lseek(s->fd, bitmap_offset + (pageentry_index / 8), SEEK_SET);
291:
292: read(s->fd, &bitmap_entry, 1);
293:
294: if ((bitmap_entry >> (pageentry_index % 8)) & 1)
295: return -1; // not allocated
296: #endif
297: #endif
298:
299: return block_offset;
300: }
301:
302: /*
303: * Writes the footer to the end of the image file. This is needed when the
304: * file grows as it overwrites the old footer
305: *
306: * Returns 0 on success and < 0 on error
307: */
308: static int rewrite_footer(BlockDriverState* bs)
309: {
310: int ret;
311: BDRVVPCState *s = bs->opaque;
312: int64_t offset = s->free_data_block_offset;
313:
1.1.1.4 ! root 314: ret = bdrv_pwrite_sync(bs->file, offset, s->footer_buf, HEADER_SIZE);
1.1 root 315: if (ret < 0)
316: return ret;
317:
318: return 0;
319: }
320:
321: /*
322: * Allocates a new block. This involves writing a new footer and updating
323: * the Block Allocation Table to use the space at the old end of the image
324: * file (overwriting the old footer)
325: *
326: * Returns the sectors' offset in the image file on success and < 0 on error
327: */
328: static int64_t alloc_block(BlockDriverState* bs, int64_t sector_num)
329: {
330: BDRVVPCState *s = bs->opaque;
331: int64_t bat_offset;
332: uint32_t index, bat_value;
333: int ret;
334: uint8_t bitmap[s->bitmap_size];
335:
336: // Check if sector_num is valid
337: if ((sector_num < 0) || (sector_num > bs->total_sectors))
338: return -1;
339:
340: // Write entry into in-memory BAT
341: index = (sector_num * 512) / s->block_size;
342: if (s->pagetable[index] != 0xFFFFFFFF)
343: return -1;
344:
345: s->pagetable[index] = s->free_data_block_offset / 512;
346:
347: // Initialize the block's bitmap
348: memset(bitmap, 0xff, s->bitmap_size);
1.1.1.4 ! root 349: bdrv_pwrite_sync(bs->file, s->free_data_block_offset, bitmap,
1.1.1.3 root 350: s->bitmap_size);
1.1 root 351:
352: // Write new footer (the old one will be overwritten)
353: s->free_data_block_offset += s->block_size + s->bitmap_size;
354: ret = rewrite_footer(bs);
355: if (ret < 0)
356: goto fail;
357:
358: // Write BAT entry to disk
359: bat_offset = s->bat_offset + (4 * index);
360: bat_value = be32_to_cpu(s->pagetable[index]);
1.1.1.4 ! root 361: ret = bdrv_pwrite_sync(bs->file, bat_offset, &bat_value, 4);
1.1 root 362: if (ret < 0)
363: goto fail;
364:
365: return get_sector_offset(bs, sector_num, 0);
366:
367: fail:
368: s->free_data_block_offset -= (s->block_size + s->bitmap_size);
369: return -1;
370: }
371:
372: static int vpc_read(BlockDriverState *bs, int64_t sector_num,
373: uint8_t *buf, int nb_sectors)
374: {
375: BDRVVPCState *s = bs->opaque;
376: int ret;
377: int64_t offset;
1.1.1.4 ! root 378: int64_t sectors, sectors_per_block;
1.1 root 379:
380: while (nb_sectors > 0) {
381: offset = get_sector_offset(bs, sector_num, 0);
382:
1.1.1.4 ! root 383: sectors_per_block = s->block_size >> BDRV_SECTOR_BITS;
! 384: sectors = sectors_per_block - (sector_num % sectors_per_block);
! 385: if (sectors > nb_sectors) {
! 386: sectors = nb_sectors;
! 387: }
! 388:
1.1 root 389: if (offset == -1) {
1.1.1.4 ! root 390: memset(buf, 0, sectors * BDRV_SECTOR_SIZE);
1.1 root 391: } else {
1.1.1.4 ! root 392: ret = bdrv_pread(bs->file, offset, buf,
! 393: sectors * BDRV_SECTOR_SIZE);
! 394: if (ret != sectors * BDRV_SECTOR_SIZE) {
1.1 root 395: return -1;
1.1.1.4 ! root 396: }
1.1 root 397: }
398:
1.1.1.4 ! root 399: nb_sectors -= sectors;
! 400: sector_num += sectors;
! 401: buf += sectors * BDRV_SECTOR_SIZE;
1.1 root 402: }
403: return 0;
404: }
405:
406: static int vpc_write(BlockDriverState *bs, int64_t sector_num,
407: const uint8_t *buf, int nb_sectors)
408: {
409: BDRVVPCState *s = bs->opaque;
410: int64_t offset;
1.1.1.4 ! root 411: int64_t sectors, sectors_per_block;
1.1 root 412: int ret;
413:
414: while (nb_sectors > 0) {
415: offset = get_sector_offset(bs, sector_num, 1);
416:
1.1.1.4 ! root 417: sectors_per_block = s->block_size >> BDRV_SECTOR_BITS;
! 418: sectors = sectors_per_block - (sector_num % sectors_per_block);
! 419: if (sectors > nb_sectors) {
! 420: sectors = nb_sectors;
! 421: }
! 422:
1.1 root 423: if (offset == -1) {
424: offset = alloc_block(bs, sector_num);
425: if (offset < 0)
426: return -1;
427: }
428:
1.1.1.4 ! root 429: ret = bdrv_pwrite(bs->file, offset, buf, sectors * BDRV_SECTOR_SIZE);
! 430: if (ret != sectors * BDRV_SECTOR_SIZE) {
1.1 root 431: return -1;
1.1.1.4 ! root 432: }
1.1 root 433:
1.1.1.4 ! root 434: nb_sectors -= sectors;
! 435: sector_num += sectors;
! 436: buf += sectors * BDRV_SECTOR_SIZE;
1.1 root 437: }
438:
439: return 0;
440: }
441:
442:
443: /*
444: * Calculates the number of cylinders, heads and sectors per cylinder
445: * based on a given number of sectors. This is the algorithm described
446: * in the VHD specification.
447: *
448: * Note that the geometry doesn't always exactly match total_sectors but
449: * may round it down.
450: *
451: * Returns 0 on success, -EFBIG if the size is larger than 127 GB
452: */
453: static int calculate_geometry(int64_t total_sectors, uint16_t* cyls,
454: uint8_t* heads, uint8_t* secs_per_cyl)
455: {
456: uint32_t cyls_times_heads;
457:
458: if (total_sectors > 65535 * 16 * 255)
459: return -EFBIG;
460:
461: if (total_sectors > 65535 * 16 * 63) {
462: *secs_per_cyl = 255;
463: *heads = 16;
464: cyls_times_heads = total_sectors / *secs_per_cyl;
465: } else {
466: *secs_per_cyl = 17;
467: cyls_times_heads = total_sectors / *secs_per_cyl;
468: *heads = (cyls_times_heads + 1023) / 1024;
469:
470: if (*heads < 4)
471: *heads = 4;
472:
473: if (cyls_times_heads >= (*heads * 1024) || *heads > 16) {
474: *secs_per_cyl = 31;
475: *heads = 16;
476: cyls_times_heads = total_sectors / *secs_per_cyl;
477: }
478:
479: if (cyls_times_heads >= (*heads * 1024)) {
480: *secs_per_cyl = 63;
481: *heads = 16;
482: cyls_times_heads = total_sectors / *secs_per_cyl;
483: }
484: }
485:
1.1.1.3 root 486: *cyls = cyls_times_heads / *heads;
1.1 root 487:
488: return 0;
489: }
490:
491: static int vpc_create(const char *filename, QEMUOptionParameter *options)
492: {
493: uint8_t buf[1024];
494: struct vhd_footer* footer = (struct vhd_footer*) buf;
495: struct vhd_dyndisk_header* dyndisk_header =
496: (struct vhd_dyndisk_header*) buf;
497: int fd, i;
1.1.1.3 root 498: uint16_t cyls = 0;
499: uint8_t heads = 0;
500: uint8_t secs_per_cyl = 0;
1.1 root 501: size_t block_size, num_bat_entries;
502: int64_t total_sectors = 0;
503:
504: // Read out options
505: while (options && options->name) {
506: if (!strcmp(options->name, "size")) {
507: total_sectors = options->value.n / 512;
508: }
509: options++;
510: }
511:
512: // Create the file
513: fd = open(filename, O_WRONLY | O_CREAT | O_TRUNC | O_BINARY, 0644);
514: if (fd < 0)
515: return -EIO;
516:
1.1.1.3 root 517: /* Calculate matching total_size and geometry. Increase the number of
518: sectors requested until we get enough (or fail). */
519: for (i = 0; total_sectors > (int64_t)cyls * heads * secs_per_cyl; i++) {
520: if (calculate_geometry(total_sectors + i,
521: &cyls, &heads, &secs_per_cyl)) {
522: return -EFBIG;
523: }
524: }
1.1 root 525: total_sectors = (int64_t) cyls * heads * secs_per_cyl;
526:
527: // Prepare the Hard Disk Footer
528: memset(buf, 0, 1024);
529:
1.1.1.2 root 530: memcpy(footer->creator, "conectix", 8);
1.1 root 531: // TODO Check if "qemu" creator_app is ok for VPC
1.1.1.2 root 532: memcpy(footer->creator_app, "qemu", 4);
533: memcpy(footer->creator_os, "Wi2k", 4);
1.1 root 534:
535: footer->features = be32_to_cpu(0x02);
536: footer->version = be32_to_cpu(0x00010000);
537: footer->data_offset = be64_to_cpu(HEADER_SIZE);
538: footer->timestamp = be32_to_cpu(time(NULL) - VHD_TIMESTAMP_BASE);
539:
540: // Version of Virtual PC 2007
541: footer->major = be16_to_cpu(0x0005);
542: footer->minor =be16_to_cpu(0x0003);
543:
544: footer->orig_size = be64_to_cpu(total_sectors * 512);
545: footer->size = be64_to_cpu(total_sectors * 512);
546:
547: footer->cyls = be16_to_cpu(cyls);
548: footer->heads = heads;
549: footer->secs_per_cyl = secs_per_cyl;
550:
551: footer->type = be32_to_cpu(VHD_DYNAMIC);
552:
553: // TODO uuid is missing
554:
555: footer->checksum = be32_to_cpu(vpc_checksum(buf, HEADER_SIZE));
556:
557: // Write the footer (twice: at the beginning and at the end)
558: block_size = 0x200000;
559: num_bat_entries = (total_sectors + block_size / 512) / (block_size / 512);
560:
561: if (write(fd, buf, HEADER_SIZE) != HEADER_SIZE)
562: return -EIO;
563:
564: if (lseek(fd, 1536 + ((num_bat_entries * 4 + 511) & ~511), SEEK_SET) < 0)
565: return -EIO;
566: if (write(fd, buf, HEADER_SIZE) != HEADER_SIZE)
567: return -EIO;
568:
569: // Write the initial BAT
570: if (lseek(fd, 3 * 512, SEEK_SET) < 0)
571: return -EIO;
572:
573: memset(buf, 0xFF, 512);
574: for (i = 0; i < (num_bat_entries * 4 + 511) / 512; i++)
575: if (write(fd, buf, 512) != 512)
576: return -EIO;
577:
578:
579: // Prepare the Dynamic Disk Header
580: memset(buf, 0, 1024);
581:
1.1.1.2 root 582: memcpy(dyndisk_header->magic, "cxsparse", 8);
1.1 root 583:
584: dyndisk_header->data_offset = be64_to_cpu(0xFFFFFFFF);
585: dyndisk_header->table_offset = be64_to_cpu(3 * 512);
586: dyndisk_header->version = be32_to_cpu(0x00010000);
587: dyndisk_header->block_size = be32_to_cpu(block_size);
588: dyndisk_header->max_table_entries = be32_to_cpu(num_bat_entries);
589:
590: dyndisk_header->checksum = be32_to_cpu(vpc_checksum(buf, 1024));
591:
592: // Write the header
593: if (lseek(fd, 512, SEEK_SET) < 0)
594: return -EIO;
595: if (write(fd, buf, 1024) != 1024)
596: return -EIO;
597:
598: close(fd);
599: return 0;
600: }
601:
602: static void vpc_close(BlockDriverState *bs)
603: {
604: BDRVVPCState *s = bs->opaque;
605: qemu_free(s->pagetable);
606: #ifdef CACHE
607: qemu_free(s->pageentry_u8);
608: #endif
609: }
610:
611: static QEMUOptionParameter vpc_create_options[] = {
612: {
613: .name = BLOCK_OPT_SIZE,
614: .type = OPT_SIZE,
615: .help = "Virtual disk size"
616: },
617: { NULL }
618: };
619:
620: static BlockDriver bdrv_vpc = {
621: .format_name = "vpc",
622: .instance_size = sizeof(BDRVVPCState),
623: .bdrv_probe = vpc_probe,
624: .bdrv_open = vpc_open,
625: .bdrv_read = vpc_read,
626: .bdrv_write = vpc_write,
627: .bdrv_close = vpc_close,
628: .bdrv_create = vpc_create,
629:
630: .create_options = vpc_create_options,
631: };
632:
633: static void bdrv_vpc_init(void)
634: {
635: bdrv_register(&bdrv_vpc);
636: }
637:
638: block_init(bdrv_vpc_init);
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.