Skip to content

Commit 9e27240

Browse files
guysoftguysoft
committed
Auto download images by base board, make python code a python package #214 
1 parent c9a0914 commit 9e27240

19 files changed

Lines changed: 300 additions & 29 deletions

‎pyproject.toml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
[tool.poetry]
2+
name = "custompios"
3+
version = "2.0.0"
4+
description = "A Raspberry Pi and other ARM devices distribution builder. CustomPiOS opens an already existing image, modifies it and repackages the image ready to ship."
5+
authors = ["Guy Sheffer <[email protected]>"]
6+
license = "GPLv3"
7+
readme = "README.rst"
8+
packages = [
9+
# { include = "src/*" },
10+
{ include = "custompios_core", from = "src" }
11+
]
12+
13+
[tool.poetry.dependencies]
14+
python = "^3.11"
15+
GitPython = "^3.1.41"
16+
17+
[tool.poetry.group.dev.dependencies]
18+
types-PyYAML = "^6.0.12.12"
19+
20+
[tool.poetry.scripts]
21+
custompios_build = 'custompios_core.multi_build:main'
22+
23+
[build-system]
24+
requires = ["poetry-core"]
25+
build-backend = "poetry.core.masonry.api"

‎src/base_image_downloader_wrapper.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,5 @@ fi
1717
# source "${DIST_PATH}/config"
1818
source "${CUSTOM_PI_OS_PATH}/config" "${WORKSPACE_SUFFIX}"
1919

20-
python3 ${CUSTOM_PI_OS_PATH}/base_image_downloader.py "${WORKSPACE_SUFFIX}"
20+
python3 ${CUSTOM_PI_OS_PATH}/custompios_core/base_image_downloader.py "${WORKSPACE_SUFFIX}"
2121

‎src/build‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,16 @@ define(){ IFS='\n' read -r -d '' ${1} || true; }
1010

1111
define SCRIPT <<'EOF'
1212
BUILD_SCRIPT_PATH=$(dirname $(realpath -s $BASH_SOURCE))
13+
export EXTRA_BOARD_CONFIG=$(mktemp)
14+
${BUILD_SCRIPT_PATH}/custompios_core/generate_board_config.py "${EXTRA_BOARD_CONFIG}"
15+
echo "Temp source file: ${EXTRA_BOARD_CONFIG}"
16+
1317
source ${BUILD_SCRIPT_PATH}/common.sh
1418
install_cleanup_trap
1519
1620
CUSTOM_OS_PATH=$(dirname $(realpath -s $0))
1721
18-
source ${CUSTOM_PI_OS_PATH}/config ${@}
22+
source ${CUSTOM_PI_OS_PATH}/config "${1}" "${EXTRA_BOARD_CONFIG}" ${@}
1923
${CUSTOM_PI_OS_PATH}/config_sanity
2024
2125
[ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios ${@}

‎src/common.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,10 +150,13 @@ function unpack() {
150150
}
151151

152152
function detach_all_loopback(){
153+
image_name=$1
153154
# Cleans up mounted loopback devices from the image name
154155
# NOTE: it might need a better way to grep for the image name, its might clash with other builds
155156
for img in $(losetup | grep $1 | awk '{ print $1 }' ); do
156-
if [ -f "${img}" ] || [ -b "${img}" ]; then
157+
# test if the image name is a substring
158+
if [ "${img}" != "$(printf '%s' "${img}" | sed 's/'"${image_name}"'//g')" ] && ([ -f "${img}" ] || [ -b "${img}" ]); then
159+
echo "freeing up $img"
157160
losetup -d $img
158161
fi
159162
done

‎src/config‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ export BUILD_VARIANT=""
77
BUILD_VARIANT="$1"
88
: ${BUILD_VARIANT:=default}
99

10+
EXTRA_BAORD_CONFIG=$2
11+
1012
export BUILD_FLAVOR=""
1113
# Disable flavor system
1214
#BUILD_FLAVOR="$1"
@@ -86,6 +88,13 @@ MODULES_LIST="${TMP//)/,}"
8688
# [ -n "$BASE_CHROOT_SCRIPT_PATH" ] || BASE_CHROOT_SCRIPT_PATH=$BASE_SCRIPT_PATH/chroot_script
8789
[ -n "$BASE_MOUNT_PATH" ] || BASE_MOUNT_PATH=$BASE_WORKSPACE/mount
8890

91+
# Import remote and submodules config
92+
if [ -f "${EXTRA_BAORD_CONFIG}" ]; then
93+
source "${EXTRA_BAORD_CONFIG}"
94+
else
95+
echo "Note: Not sourceing board config"
96+
fi
97+
8998
export REMOTE_AND_META_CONFIG="$BASE_WORKSPACE"/remote_and_meta_config
9099
# Remote modules and meta modulese go in first if they want to change standard behaviour
91100
if [ -f "${REMOTE_AND_META_CONFIG}" ]; then

‎src/custompios‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,11 @@ fi
116116
mkdir -p $BASE_WORKSPACE
117117
mkdir -p $BASE_MOUNT_PATH
118118

119+
# This is already genrated at "build" sourced in "config", but copying here mostly for debug
120+
if [ -f "${EXTRA_BAORD_CONFIG}" ]; then
121+
mv -v "${EXTRA_BAORD_CONFIG}" "${BASE_WORKSPACE}"/extra_board_config
122+
fi
123+
119124
# Clean exported artifacts from other builds
120125
rm -rf "${BASE_WORKSPACE}"/*.tar.gz
121126

@@ -129,6 +134,7 @@ pushd $BASE_WORKSPACE
129134
fi
130135
if [ ! -f "$BASE_ZIP_IMG" ] || [ "$BASE_ZIP_IMG" == "" ]; then
131136
echo "Error: could not find image: $BASE_ZIP_IMG"
137+
echo "On CustomPiOS v2 you can provide -d to download the latest image of your board automatically"
132138
exit 1
133139
fi
134140

@@ -189,7 +195,7 @@ pushd $BASE_WORKSPACE
189195
CHROOT_SCRIPT=${BASE_WORKSPACE}/chroot_script
190196
MODULES_AFTER_PATH=${BASE_WORKSPACE}/modules_after
191197
MODULES_BEFORE="${MODULES}"
192-
${CUSTOM_PI_OS_PATH}/execution_order.py "${MODULES}" "${CHROOT_SCRIPT}" "${MODULES_AFTER_PATH}" "${REMOTE_AND_META_CONFIG}"
198+
${CUSTOM_PI_OS_PATH}/custompios_core/execution_order.py "${MODULES}" "${CHROOT_SCRIPT}" "${MODULES_AFTER_PATH}" "${REMOTE_AND_META_CONFIG}"
193199
if [ -f "${REMOTE_AND_META_CONFIG}" ]; then
194200
echo "Sourcing remote and submodules config"
195201
source "${REMOTE_AND_META_CONFIG}" ${@}

‎src/custompios_core/__init__.py‎

Whitespace-only changes.

src/base_image_downloader.py renamed to src/custompios_core/base_image_downloader.py

Lines changed: 77 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,13 @@
77
import hashlib
88
import shutil
99
import re
10+
from typing import Dict, Any, Optional, cast, Tuple
11+
from common import get_image_config, read_images
1012
PRECENT_PROGRESS_SIZE = 5
1113

1214
class ChecksumFailException(Exception):
1315
pass
1416

15-
IMAGES_CONFIG = os.path.join(os.path.dirname(__file__), "images.yml")
1617
RETRY = 3
1718

1819
def ensure_dir(d, chmod=0o777):
@@ -26,13 +27,31 @@ def ensure_dir(d, chmod=0o777):
2627
return False
2728
return True
2829

29-
def read_images():
30-
if not os.path.isfile(IMAGES_CONFIG):
31-
raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}")
32-
with open(IMAGES_CONFIG,'r') as f:
33-
output = yaml.safe_load(f)
34-
return output
3530

31+
def download_webpage(url: str) -> Optional[str]:
32+
try:
33+
with urllib.request.urlopen(url) as response:
34+
# Decode the response to a string
35+
webpage = response.read().decode('utf-8')
36+
return webpage
37+
except Exception as e:
38+
print(str(e))
39+
return None
40+
41+
def get_location_header(url: str) -> str:
42+
try:
43+
with urllib.request.urlopen(url) as response:
44+
response_url = response.url
45+
46+
if response_url is None:
47+
raise Exception("Location header is None, can't determine latest rpi image")
48+
return response_url
49+
except Exception as e:
50+
print(str(e))
51+
print("Error: Failed to determine latest rpi image")
52+
raise e
53+
54+
3655
class DownloadProgress:
3756
last_precent: float = 0
3857
def show_progress(self, block_num, block_size, total_size):
@@ -41,8 +60,10 @@ def show_progress(self, block_num, block_size, total_size):
4160
print(f"{new_precent}%", end="\r")
4261
self.last_precent = new_precent
4362

44-
def get_file_name(headers):
45-
return re.findall("filename=(\S+)", headers["Content-Disposition"])[0]
63+
def get_file_name(headers, url):
64+
if "Content-Disposition" in headers.keys():
65+
return re.findall("filename=(\S+)", headers["Content-Disposition"])[0]
66+
return url.split('/')[-1]
4667

4768
def get_sha256(filename):
4869
sha256_hash = hashlib.sha256()
@@ -53,10 +74,12 @@ def get_sha256(filename):
5374
return file_checksum
5475
return
5576

56-
def download_image_http(board: str, dest_folder: str, redownload: bool = False):
77+
def download_image_http(board: Dict[str, Any], dest_folder: str, redownload: bool = False):
5778
url = board["url"]
5879
checksum = board["checksum"]
80+
download_http(url, checksum)
5981

82+
def download_http(url: str, checksum_url: str, dest_folder: str, redownload: bool = False):
6083
with tempfile.TemporaryDirectory() as tmpdirname:
6184
print('created temporary directory', tmpdirname)
6285
temp_file_name = os.path.join(tmpdirname, "image.xz")
@@ -66,8 +89,8 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
6689
try:
6790
# Get sha and confirm its the right image
6891
download_progress = DownloadProgress()
69-
_, headers_checksum = urllib.request.urlretrieve(checksum, temp_file_checksum, download_progress.show_progress)
70-
file_name_checksum = get_file_name(headers_checksum)
92+
_, headers_checksum = urllib.request.urlretrieve(checksum_url, temp_file_checksum, download_progress.show_progress)
93+
file_name_checksum = get_file_name(headers_checksum, checksum_url)
7194

7295
checksum_data = None
7396
with open(temp_file_checksum, 'r') as f:
@@ -82,13 +105,13 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
82105
if os.path.isfile(dest_file_name):
83106
file_checksum = get_sha256(dest_file_name)
84107
if file_checksum == online_checksum:
85-
# We got file and checksum is right
108+
print("We got base image file and checksum is right")
86109
return
87110
# Get the file
88111
download_progress = DownloadProgress()
89112
_, headers = urllib.request.urlretrieve(url, temp_file_name, download_progress.show_progress)
90113

91-
file_name = get_file_name(headers)
114+
file_name = get_file_name(headers, url)
92115
file_checksum = get_sha256(temp_file_name)
93116
if file_checksum != online_checksum:
94117
print(f'Failed. Attempt # {r + 1}, checksum missmatch: {file_checksum} expected: {online_checksum}')
@@ -102,11 +125,31 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
102125
else:
103126
print('Error encoutered at {RETRY} attempt')
104127
print(e)
128+
exit(1)
105129
else:
106130
print(f"Success: {temp_file_name}")
107131
break
108132
return
109133

134+
135+
def download_image_rpi(board: Dict[str, Any], dest_folder: str):
136+
port = board.get("port", "lite_armhf")
137+
os_name = f"raspios"
138+
distribution = board.get("distribution", "bookworm")
139+
version_file = board.get("version_file", "latest")
140+
version_folder = board.get("version_folder", "latest")
141+
142+
latest_url = f"https://downloads.raspberrypi.org/{os_name}_{port}_latest"
143+
144+
download_url = f"https://downloads.raspberrypi.org/{os_name}_{port}/images/{os_name}_{port}-{version_folder}/{version_file}-{os_name}-{distribution}-{port}.img.xz"
145+
if version_file == "latest" or version_folder == "latest":
146+
download_url = get_location_header(latest_url)
147+
148+
checksum_url = f"{download_url}.sha256"
149+
download_http(download_url, checksum_url, dest_folder)
150+
return
151+
152+
110153
if __name__ == "__main__":
111154
parser = argparse.ArgumentParser(add_help=True, description='Download images based on BASE_BOARD and BASE_O')
112155
parser.add_argument('WORKSPACE_SUFFIX', nargs='?', default="default", help="The workspace folder suffix used folder")
@@ -118,14 +161,28 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
118161
base_board = os.environ.get("BASE_BOARD", None)
119162
base_image_path = os.environ.get("BASE_IMAGE_PATH", None)
120163

121-
if base_board is not None and base_board in images["images"]:
122-
if images["images"][base_board]["type"] == "http":
123-
download_image_http(images["images"][base_board], base_image_path)
124-
elif images["images"][base_board]["type"] == "torrent":
164+
if base_image_path is None:
165+
print(f'Error: did not find image config file')
166+
exit(1)
167+
cast(str, base_image_path)
168+
169+
image_config = get_image_config()
170+
if image_config is not None:
171+
if image_config["type"] == "http":
172+
print(f"Downloading image for {base_board}")
173+
download_image_http(image_config, base_image_path)
174+
elif image_config["type"] == "rpi":
175+
print(f"Downloading Raspberry Pi image for {base_board}")
176+
download_image_rpi(image_config, base_image_path)
177+
elif image_config["type"] == "torrent":
125178
print("Error: Torrent not implemented")
126179
exit(1)
127180
else:
128-
print("Error: Unsupported image download type")
181+
print(f'Error: Unsupported image download type: {image_config["type"]}')
129182
exit(1)
183+
else:
184+
print(f"Error: Image config not found for: {base_board}")
185+
exit(1)
186+
130187

131-
print("Done")
188+
print("Done")

‎src/custompios_core/common.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
""" Common functions between CustomPiOS python scripts"""
2+
from typing import Dict, Any, Optional, cast
3+
import yaml
4+
import os
5+
from pathlib import Path
6+
7+
def get_custompios_folder():
8+
custompios_path = os.environ.get("CUSTOM_PI_OS_PATH", None)
9+
if custompios_path is not None:
10+
return Path(custompios_path)
11+
return Path(__file__).parent.parent
12+
13+
14+
IMAGES_CONFIG = os.path.join(get_custompios_folder(), "images.yml")
15+
16+
17+
def read_images() -> Dict[str, Dict[str,str]]:
18+
if not os.path.isfile(IMAGES_CONFIG):
19+
raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}")
20+
with open(IMAGES_CONFIG,'r') as f:
21+
output = yaml.safe_load(f)
22+
return output
23+
24+
def get_image_config() -> Optional[Dict["str", Any]]:
25+
images = read_images()
26+
27+
base_board = os.environ.get("BASE_BOARD", None)
28+
base_image_path = os.environ.get("BASE_IMAGE_PATH", None)
29+
30+
if base_board is not None and base_board in images["images"]:
31+
return images["images"][base_board]
32+
return None

src/execution_order.py renamed to src/custompios_core/execution_order.py

File renamed without changes.

0 commit comments

Comments
 (0)