Readable source
#!/usr/bin/env python3
"""Report byte-identical files using SHA-256. Never deletes or modifies files."""
import argparse,json,hashlib
from pathlib import Path
def digest(path):
h=hashlib.sha256()
with path.open('rb') as f:
for chunk in iter(lambda:f.read(1024*1024),b''):h.update(chunk)
return h.hexdigest()
def main():
p=argparse.ArgumentParser(description=__doc__);p.add_argument('folder',type=Path);a=p.parse_args()
if not a.folder.is_dir():p.error('Folder does not exist.')
groups={};errors=[];root=a.folder.resolve()
for f in sorted(root.rglob('*')):
if f.is_file() and not f.is_symlink():
try:groups.setdefault((f.stat().st_size,digest(f)),[]).append(str(f.relative_to(root)))
except OSError as e:errors.append({'path':str(f),'error':str(e)})
print(json.dumps({'duplicates':[{'bytes':n,'sha256':sha,'files':fs} for (n,sha),fs in groups.items() if len(fs)>1],'errors':errors},indent=2))
if __name__=='__main__':main()