Browse Source

download_hash: cache request for 'multi-hash' files

This avoid re-downloading the same file for different arch and
re-parsing it
pull/11513/head
Max Gautier 2 months ago
parent
commit
dec4e711d1
Failed to extract signature
1 changed files with 31 additions and 13 deletions
  1. 44
      scripts/download_hash.py

44
scripts/download_hash.py

@ -8,6 +8,7 @@ import sys
from itertools import count, groupby from itertools import count, groupby
from collections import defaultdict from collections import defaultdict
from functools import cache
import argparse import argparse
import requests import requests
from ruamel.yaml import YAML from ruamel.yaml import YAML
@ -100,6 +101,20 @@ def download_hash(only_downloads: [str]) -> None:
data, yaml = open_checksums_yaml() data, yaml = open_checksums_yaml()
s = requests.Session() s = requests.Session()
@cache
def _get_hash_by_arch(download: str, version: str) -> {str: str}:
hash_file = s.get(downloads[download].format(
version = version,
os = "linux",
),
allow_redirects=True)
if hash_file.status_code == 404:
print(f"Unable to find {download} hash file for version {version} at {hash_file.url}")
return None
hash_file.raise_for_status()
return download_hash_extract[download](hash_file.content.decode())
for download, url in (downloads if only_downloads == [] for download, url in (downloads if only_downloads == []
else {k:downloads[k] for k in downloads.keys() & only_downloads}).items(): else {k:downloads[k] for k in downloads.keys() & only_downloads}).items():
checksum_name = f"{download}_checksums" checksum_name = f"{download}_checksums"
@ -125,22 +140,25 @@ def download_hash(only_downloads: [str]) -> None:
# to find new versions # to find new versions
if version in versions and versions[version] != 0: if version in versions and versions[version] != 0:
continue continue
hash_file = s.get(downloads[download].format(
version = version,
os = "linux",
arch = arch
),
allow_redirects=True)
if hash_file.status_code == 404:
print(f"Unable to find {download} hash file for version {version} (arch: {arch}) at {hash_file.url}")
break
hash_file.raise_for_status()
sha256sum = hash_file.content.decode()
if download in download_hash_extract: if download in download_hash_extract:
sha256sum = download_hash_extract[download](sha256sum).get(arch)
hashes = _get_hash_by_arch(download, version)
if hashes == None:
break
sha256sum = hashes.get(arch)
if sha256sum == None: if sha256sum == None:
break break
sha256sum = sha256sum.split()[0]
else:
hash_file = s.get(downloads[download].format(
version = version,
os = "linux",
arch = arch
),
allow_redirects=True)
if hash_file.status_code == 404:
print(f"Unable to find {download} hash file for version {version} (arch: {arch}) at {hash_file.url}")
break
hash_file.raise_for_status()
sha256sum = hash_file.content.decode().split()[0]
if len(sha256sum) != 64: if len(sha256sum) != 64:
raise Exception(f"Checksum has an unexpected length: {len(sha256sum)} (binary: {download}, arch: {arch}, release: {version}, checksum: '{sha256sum}')") raise Exception(f"Checksum has an unexpected length: {len(sha256sum)} (binary: {download}, arch: {arch}, release: {version}, checksum: '{sha256sum}')")

Loading…
Cancel
Save