![[BACK]](/cvsweb/icons/back.gif){kind=icon}
Return to finddupes.c CVS log ![[TXT]](/cvsweb/icons/text.gif){kind=icon}
![[DIR]](/cvsweb/icons/dir.gif){kind=icon}
Up to [Synchronet] / sbbs / sbbs3|
Return to finddupes.c CVS log | Up to [Synchronet] / sbbs / sbbs3 |
1.1 root 1: /* finddupes.c */
2:
3: /* Search for (and optionally delete) duplicate files in multiple */
4: /* directories based on size and either MD5 or CRC-32 "chksums". */
5:
6: #include <stdio.h>
7: #include <time.h>
8:
9: #include "dirwrap.h"
10: #ifdef USE_MD5
11: #include "md5.h"
12: #else /* CRC-32 */
13: #include "crc32.h"
14: #endif
15:
16: typedef struct {
17: char path[MAX_PATH+1];
18: long length;
19: time_t date;
20: #ifdef USE_MD5
21: BYTE chksum[MD5_DIGEST_SIZE];
22: #else
23: ulong chksum;
24: #endif
25: } file_t;
26:
27: file_t* file;
28: ulong file_count=0;
29:
30: #ifdef FREE
31: #undef FREE
32: #endif
33: #define FREE(x) if(x!=NULL) free(x);
34:
35: int fchksum(const char* fname, long length,
36: #ifdef USE_MD5
37: BYTE*
38: #else
39: ulong*
40: #endif
41: chksum)
42: {
43: BYTE* buf=NULL;
44: FILE* fp;
45:
46: if((fp=fopen(fname,"rb"))==NULL) {
47: perror(fname);
48: return(-1);
49: }
50:
51: if(length && (buf=malloc(length))==NULL) {
52: printf("!Error allocating %ld bytes of memory for %s\n"
53: ,length,fname);
54: fclose(fp);
55: return(-1);
56: }
57:
58: if(fread(buf,sizeof(BYTE),length,fp) != length) {
59: perror(fname);
60: fclose(fp);
61: FREE(buf);
62: return(-1);
63: }
64:
65: fclose(fp);
66: #ifdef USE_MD5
67: MD5_calc(chksum, buf, length);
68: #else
69: *chksum = crc32(buf, length);
70: #endif
71: FREE(buf);
72: return(0);
73: }
74:
75: char* timestr(void)
76: {
77: char* p;
78: time_t t=time(NULL);
79: p=ctime(&t);
80: p[19]=0; /* chop off year and \n */
81: return(p+4); /* skip day-of-week */
82: }
83:
84: int searchdir(const char* path, BOOL recursive, ulong compare_bytes)
85: {
86: DIR* dir;
87: struct dirent* ent;
88: file_t* fp;
89: char fpath[MAX_PATH+1];
90:
91: printf("%s begin searching %s\n",timestr(), path);
92: if((dir = opendir(path))==NULL) {
93: perror(path);
94: return(1);
95: }
96:
97: while((ent = readdir(dir))!=NULL) {
98: if(kbhit())
99: break;
100: if(strcmp(ent->d_name,".")==0 || strcmp(ent->d_name,"..")==0)
101: continue;
102: strcpy(fpath,path);
103: backslash(fpath);
104: strcat(fpath,ent->d_name);
105: if(isdir(fpath)) {
106: if(recursive)
107: searchdir(fpath, recursive, compare_bytes);
108: continue;
109: }
110:
111: file=realloc(file,sizeof(file_t)*(file_count+1));
112: if(file==NULL) {
113: printf("!Error allocating %lu bytes\n",sizeof(file_t)*(file_count+1));
114: exit(1);
115: }
116: fp=&file[file_count];
117: memset(fp,0,sizeof(file_t));
118: strcpy(fp->path,fpath);
119: fp->date=fdate(fp->path);
120: if((fp->length=flength(fp->path))==-1) {
121: printf("!Failed to get length of %s\n",fp->path);
122: continue;
123: }
124: if(compare_bytes && fp->length > compare_bytes)
125: fp->length = compare_bytes;
126: if(fchksum(fp->path, fp->length,
127: #ifdef USE_MD5
128: fp->chksum
129: #else
130: &fp->chksum
131: #endif
132: ))
133: continue;
134: file_count++;
135: printf("%lu\r", file_count);
136: }
137:
138: closedir(dir);
139: printf("%s done searching %s\n",timestr(), path);
140: return(0);
141: }
142:
143: int compare_files(const file_t *f1, const file_t *f2 )
144: {
145: int result;
146:
147: /* Sort first by size (descending) */
148: if((result = f2->length - f1->length) != 0)
149: return(result);
150:
151: /* Then by chksum (ascending) */
152: if((result = memcmp(&f1->chksum, &f2->chksum, sizeof(f1->chksum))) != 0)
153: return(result);
154:
155: /* Then by date (descending) */
156: return(f2->date - f1->date);
157: }
158:
159: int main(int argc, char** argv)
160: {
161: char hex[32];
162: int i;
163: ulong fsize;
164: ulong dupe_count=0;
165: ulong del_files=0;
166: ulong del_bytes=0;
167: ulong compare_bytes=0;
168: BOOL recursive=FALSE;
169: BOOL del_dupes=FALSE;
170: BOOL dir_specified=FALSE;
171:
172: for(i=1;i<argc;i++) {
173: if(!stricmp(argv[i],"-d"))
174: del_dupes=TRUE;
175: else if(!stricmp(argv[i],"-r"))
176: recursive=TRUE;
177: else if(!stricmp(argv[i],"-b") && i<argc+1)
178: compare_bytes=atoi(argv[++i]);
179: else if(!stricmp(argv[i],"-k") && i<argc+1)
180: compare_bytes=atoi(argv[++i])*1024;
181: else if(argv[i][0]=='-') {
182: printf("%s [[-opt] [-opt] [...]] [[path] [path] [...]]\n", argv[0]);
183: printf("-r\t search directories recursively\n");
184: printf("-d\t delete duplicate files found\n");
185: printf("-b n\t compare up to n bytes of each file\n");
186: printf("-k n\t compare up to n kilobytes each of file\n");
187: exit(0);
188: }
189: else {
190: dir_specified=TRUE;
191: searchdir(argv[i], recursive, compare_bytes);
192: }
193: }
194: if(!dir_specified)
195: searchdir(".", recursive, compare_bytes);
196:
197: if(!file_count) {
198: printf("no files.\n");
199: return(0);
200: }
201:
202: printf("%s begin sorting (%lu files)\n", timestr(), file_count);
203: qsort(file,file_count,sizeof(file_t),compare_files);
204: printf("%s end sorting\n", timestr());
205:
206: printf("%s comparing (%lu files)\n", timestr(), file_count);
207:
208: for(i=0;i<file_count-1;i++) {
209: if(file[i].length != file[i+1].length)
210: continue; /* sizes must match */
211: if(memcmp(&file[i].chksum, &file[i+1].chksum, sizeof(file[i].chksum)))
212: continue; /* chksums must match */
213: #ifdef USE_MD5
214: MD5_hex(hex, file[i].chksum);
215: #else
216: sprintf(hex, "%08lx", file[i].chksum);
217: #endif
218: printf("Dupe: %s %7lu %s\n", hex, file[i].length, getfname(file[i].path));
219: if(del_dupes) {
220: fsize=flength(file[i].path);
221: printf("Removing %s (%lu bytes)\n", file[i].path, fsize);
222: if(remove(file[i].path)!=0)
223: perror(file[i].path);
224: else {
225: del_files++;
226: del_bytes+=fsize;
227: }
228: }
229: dupe_count++;
230: }
231:
232: printf("%s done (%lu duplicates found)\n", timestr(), dupe_count);
233: if(del_files)
234: printf("%lu bytes deleted in %lu files\n", del_bytes, del_files);
235: return(0);
236: }
This archive runs on limited infrastructure. Preserving old code on modern bandwidth. Automated agents are requested to crawl responsibly.