Meta: Add support for verifying downloaded files with their SHA-256 hash

Author: https://github.com/trflynn89
Commit: ec63f442f9
Pull-request: https://github.com/SerenityOS/serenity/pull/24407
Issue: https://github.com/SerenityOS/serenity/issues/24401
Reviewed-by: https://github.com/nico ✅

Meta/CMake/utils.cmake

@@ -200,6 +200,12 @@ function(invoke_generator name generator primary_source header implementation)
 endfunction()
 
 function(download_file_multisource urls path)
+    cmake_parse_arguments(DOWNLOAD "" "SHA256" "" ${ARGN})
+
+    if (NOT "${DOWNLOAD_SHA256}" STREQUAL "")
+        set(DOWNLOAD_SHA256 EXPECTED_HASH "SHA256=${DOWNLOAD_SHA256}")
+    endif()
+
     if (NOT EXISTS "${path}")
         if (NOT ENABLE_NETWORK_DOWNLOADS)
             message(FATAL_ERROR "${path} does not exist, and unable to download it")
@@ -209,7 +215,7 @@ function(download_file_multisource urls path)
         foreach(url ${urls})
             message(STATUS "Downloading file ${file} from ${url}")
 
-            file(DOWNLOAD "${url}" "${path}" INACTIVITY_TIMEOUT 10 STATUS download_result)
+            file(DOWNLOAD "${url}" "${path}" INACTIVITY_TIMEOUT 10 STATUS download_result ${DOWNLOAD_SHA256})
             list(GET download_result 0 status_code)
             list(GET download_result 1 error_message)
 
@@ -228,8 +234,10 @@ function(download_file_multisource urls path)
 endfunction()
 
 function(download_file url path)
+    cmake_parse_arguments(DOWNLOAD "" "SHA256" "" ${ARGN})
+
     # If the timestamp doesn't match exactly, the Web Archive should redirect to the closest archived file automatically.
-    download_file_multisource("${url};https://web.archive.org/web/99991231235959/${url}" "${path}")
+    download_file_multisource("${url};https://web.archive.org/web/99991231235959/${url}" "${path}" SHA256 "${DOWNLOAD_SHA256}")
 endfunction()
 
 function(extract_path dest_dir zip_path source_path dest_path)

Meta/gn/build/download_file.gni

@@ -18,6 +18,8 @@
 #   cache [String]
 #       Directory to clear on version mismatch
 #
+#   sha256 [String]
+#       Expected SHA-256 hash of the downloaded file
 #
 # Example use:
 #
@@ -66,6 +68,12 @@ template("download_file") {
         rebase_path(invoker.cache, root_build_dir),
       ]
     }
+    if (defined(invoker.sha256)) {
+      args += [
+        "-s",
+        invoker.sha256,
+      ]
+    }
 
     forward_variables_from(invoker,
                            [

Meta/gn/build/download_file.py

@@ -8,6 +8,7 @@ It's intended to be used for files that are cached between runs.
 """
 
 import argparse
+import hashlib
 import os
 import pathlib
 import shutil
@@ -16,6 +17,20 @@ import tempfile
 import urllib.request
 
 
+def compute_sha256(path):
+    sha256 = hashlib.sha256()
+
+    with open(path, 'rb') as file:
+        while True:
+            data = file.read(256 << 10)
+            if not data:
+                break
+
+            sha256.update(data)
+
+    return sha256.hexdigest()
+
+
 def main():
     parser = argparse.ArgumentParser(
         epilog=__doc__,
@@ -29,6 +44,8 @@ def main():
                         help='filesystem location to cache version')
     parser.add_argument('-c', "--cache-path", required=False,
                         help='path for cached files to clear on version mismatch')
+    parser.add_argument('-s', "--sha256", required=False,
+                        help='expected SHA-256 hash of the downloaded file')
     args = parser.parse_args()
 
     version_from_file = ''
@@ -41,23 +58,30 @@ def main():
         return 0
 
     # Fresh build or version mismatch, delete old cache
-    if (args.cache_path):
+    if args.cache_path:
         cache_path = pathlib.Path(args.cache_path)
         shutil.rmtree(cache_path, ignore_errors=True)
         cache_path.mkdir(parents=True)
 
-    print(f"Downloading version {args.version} of {args.output}...", end='')
+    output_file = pathlib.Path(args.output)
+    print(f"Downloading file {output_file} from {args.url}")
 
     with urllib.request.urlopen(args.url) as f:
         try:
-            with tempfile.NamedTemporaryFile(delete=False,
+            with tempfile.NamedTemporaryFile(delete=False, dir=output_file.parent) as out:
-                                             dir=pathlib.Path(args.output).parent) as out:
                 out.write(f.read())
-                os.rename(out.name, args.output)
+                os.rename(out.name, output_file)
         except IOError:
             os.unlink(out.name)
 
-    print("done")
+    if args.sha256:
+        actual_sha256 = compute_sha256(output_file)
+
+        if args.sha256 != actual_sha256:
+            print(f"SHA-256 mismatch for downloaded file {output_file}")
+            print(f"Expected: {args.sha256}")
+            print(f"Actual:   {actual_sha256}")
+            return 1
 
     with open(version_file, 'w') as f:
         f.write(args.version)