From ec63f442f9e0a357f932f770fdd017c07850f57d Mon Sep 17 00:00:00 2001 From: Timothy Flynn Date: Thu, 23 May 2024 12:04:02 -0400 Subject: [PATCH] Meta: Add support for verifying downloaded files with their SHA-256 hash --- Meta/CMake/utils.cmake | 12 ++++++++-- Meta/gn/build/download_file.gni | 8 +++++++ Meta/gn/build/download_file.py | 40 ++++++++++++++++++++++++++------- 3 files changed, 50 insertions(+), 10 deletions(-) diff --git a/Meta/CMake/utils.cmake b/Meta/CMake/utils.cmake index 4ebdbbd997a..42647ffd7ee 100644 --- a/Meta/CMake/utils.cmake +++ b/Meta/CMake/utils.cmake @@ -200,6 +200,12 @@ function(invoke_generator name generator primary_source header implementation) endfunction() function(download_file_multisource urls path) + cmake_parse_arguments(DOWNLOAD "" "SHA256" "" ${ARGN}) + + if (NOT "${DOWNLOAD_SHA256}" STREQUAL "") + set(DOWNLOAD_SHA256 EXPECTED_HASH "SHA256=${DOWNLOAD_SHA256}") + endif() + if (NOT EXISTS "${path}") if (NOT ENABLE_NETWORK_DOWNLOADS) message(FATAL_ERROR "${path} does not exist, and unable to download it") @@ -209,7 +215,7 @@ function(download_file_multisource urls path) foreach(url ${urls}) message(STATUS "Downloading file ${file} from ${url}") - file(DOWNLOAD "${url}" "${path}" INACTIVITY_TIMEOUT 10 STATUS download_result) + file(DOWNLOAD "${url}" "${path}" INACTIVITY_TIMEOUT 10 STATUS download_result ${DOWNLOAD_SHA256}) list(GET download_result 0 status_code) list(GET download_result 1 error_message) @@ -228,8 +234,10 @@ function(download_file_multisource urls path) endfunction() function(download_file url path) + cmake_parse_arguments(DOWNLOAD "" "SHA256" "" ${ARGN}) + # If the timestamp doesn't match exactly, the Web Archive should redirect to the closest archived file automatically. - download_file_multisource("${url};https://web.archive.org/web/99991231235959/${url}" "${path}") + download_file_multisource("${url};https://web.archive.org/web/99991231235959/${url}" "${path}" SHA256 "${DOWNLOAD_SHA256}") endfunction() function(extract_path dest_dir zip_path source_path dest_path) diff --git a/Meta/gn/build/download_file.gni b/Meta/gn/build/download_file.gni index a05a91fb70a..ff73d986f03 100644 --- a/Meta/gn/build/download_file.gni +++ b/Meta/gn/build/download_file.gni @@ -18,6 +18,8 @@ # cache [String] # Directory to clear on version mismatch # +# sha256 [String] +# Expected SHA-256 hash of the downloaded file # # Example use: # @@ -66,6 +68,12 @@ template("download_file") { rebase_path(invoker.cache, root_build_dir), ] } + if (defined(invoker.sha256)) { + args += [ + "-s", + invoker.sha256, + ] + } forward_variables_from(invoker, [ diff --git a/Meta/gn/build/download_file.py b/Meta/gn/build/download_file.py index ffeef7c4119..bf3eda82b95 100644 --- a/Meta/gn/build/download_file.py +++ b/Meta/gn/build/download_file.py @@ -8,6 +8,7 @@ It's intended to be used for files that are cached between runs. """ import argparse +import hashlib import os import pathlib import shutil @@ -16,10 +17,24 @@ import tempfile import urllib.request +def compute_sha256(path): + sha256 = hashlib.sha256() + + with open(path, 'rb') as file: + while True: + data = file.read(256 << 10) + if not data: + break + + sha256.update(data) + + return sha256.hexdigest() + + def main(): parser = argparse.ArgumentParser( - epilog=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) + epilog=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) parser.add_argument('url', help='input url') parser.add_argument('-o', '--output', required=True, help='output file') @@ -29,6 +44,8 @@ def main(): help='filesystem location to cache version') parser.add_argument('-c', "--cache-path", required=False, help='path for cached files to clear on version mismatch') + parser.add_argument('-s', "--sha256", required=False, + help='expected SHA-256 hash of the downloaded file') args = parser.parse_args() version_from_file = '' @@ -41,23 +58,30 @@ def main(): return 0 # Fresh build or version mismatch, delete old cache - if (args.cache_path): + if args.cache_path: cache_path = pathlib.Path(args.cache_path) shutil.rmtree(cache_path, ignore_errors=True) cache_path.mkdir(parents=True) - print(f"Downloading version {args.version} of {args.output}...", end='') + output_file = pathlib.Path(args.output) + print(f"Downloading file {output_file} from {args.url}") with urllib.request.urlopen(args.url) as f: try: - with tempfile.NamedTemporaryFile(delete=False, - dir=pathlib.Path(args.output).parent) as out: + with tempfile.NamedTemporaryFile(delete=False, dir=output_file.parent) as out: out.write(f.read()) - os.rename(out.name, args.output) + os.rename(out.name, output_file) except IOError: os.unlink(out.name) - print("done") + if args.sha256: + actual_sha256 = compute_sha256(output_file) + + if args.sha256 != actual_sha256: + print(f"SHA-256 mismatch for downloaded file {output_file}") + print(f"Expected: {args.sha256}") + print(f"Actual: {actual_sha256}") + return 1 with open(version_file, 'w') as f: f.write(args.version)