diff --git a/scripts/copy_collection_files.py b/scripts/copy_collection_files.py index 22ae352a..631862eb 100644 --- a/scripts/copy_collection_files.py +++ b/scripts/copy_collection_files.py @@ -1,4 +1,5 @@ import argparse +import hashlib import json import os import shutil @@ -8,7 +9,15 @@ sys.setdefaultencoding("utf-8") -def copy_collection_file(dump_files, destination_prefix, working_dir): +def file_md5(path): + md5 = hashlib.md5() + with open(path, "rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + md5.update(chunk) + return md5.hexdigest() + + +def copy_collection_file(dump_files, destination_prefix, working_dir, skip_plots=False): file_log = open(os.path.join(working_dir, "files.log"), "wb") for i, dump_file in enumerate(dump_files): @@ -23,8 +32,10 @@ def copy_collection_file(dump_files, destination_prefix, working_dir): for record in data: legacy_record_files = record["files"] recid = record["recid"] + print("Processing {}".format(recid)) for legacy_record_file in legacy_record_files: - print("Processing {}".format(recid)) + if skip_plots and legacy_record_file.get("type") == "Plot": + continue full_path = legacy_record_file["full_path"] # important: last slash path_to_replace = "/opt/cdsweb/var/data/files/" @@ -42,6 +53,16 @@ def copy_collection_file(dump_files, destination_prefix, working_dir): destination_path = destination_path.encode("utf-8") print(filename) print(destination_path) + + expected = legacy_record_file.get("checksum") or "" + dest_md5 = file_md5(destination_path) + checksum_mismatch = dest_md5 != expected + if checksum_mismatch: + raise ValueError( + "ERROR checksum mismatch: recid={} expected={} dest={}".format( + recid, expected, dest_md5 + ) + ) file_log.write( "RECID: %s bibdocid: %s file: %s, destination: %s \n" % ( @@ -64,6 +85,7 @@ def get_dump_files_paths(json_dump_dir): collection = "lep/aleph_drafts" environment = "dev" +skip_plots = False destination_prefix = "/eos/media/cds/cds-rdm/{0}/migration/{1}/files".format( environment, collection @@ -74,4 +96,4 @@ def get_dump_files_paths(json_dump_dir): ) dump_files = get_dump_files_paths(json_dump_dir) -copy_collection_file(dump_files, destination_prefix, working_dir) +copy_collection_file(dump_files, destination_prefix, working_dir, skip_plots=skip_plots)