Skip to content

Commit af3bd56

Browse files
guysoftguysoft
committed
Auto download images by base board, make python code a python package #214 
1 parent ae87c48 commit af3bd56

19 files changed

Lines changed: 244 additions & 28 deletions

‎pyproject.toml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
[tool.poetry]
2+
name = "custompios"
3+
version = "2.0.0"
4+
description = "A Raspberry Pi and other ARM devices distribution builder. CustomPiOS opens an already existing image, modifies it and repackages the image ready to ship."
5+
authors = ["Guy Sheffer <[email protected]>"]
6+
license = "GPLv3"
7+
readme = "README.rst"
8+
packages = [
9+
# { include = "src/*" },
10+
{ include = "custompios_core", from = "src" }
11+
]
12+
13+
[tool.poetry.dependencies]
14+
python = "^3.11"
15+
GitPython = "^3.1.41"
16+
17+
[tool.poetry.group.dev.dependencies]
18+
types-PyYAML = "^6.0.12.12"
19+
20+
[tool.poetry.scripts]
21+
custompios_build = 'custompios_core.multi_build:main'
22+
23+
[build-system]
24+
requires = ["poetry-core"]
25+
build-backend = "poetry.core.masonry.api"

‎src/base_image_downloader_wrapper.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,5 +17,5 @@ fi
1717
# source "${DIST_PATH}/config"
1818
source "${CUSTOM_PI_OS_PATH}/config" "${WORKSPACE_SUFFIX}"
1919

20-
python3 ${CUSTOM_PI_OS_PATH}/base_image_downloader.py "${WORKSPACE_SUFFIX}"
20+
python3 ${CUSTOM_PI_OS_PATH}/custompios_core/base_image_downloader.py "${WORKSPACE_SUFFIX}"
2121

‎src/build‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,12 +10,16 @@ define(){ IFS='\n' read -r -d '' ${1} || true; }
1010

1111
define SCRIPT <<'EOF'
1212
BUILD_SCRIPT_PATH=$(dirname $(realpath -s $BASH_SOURCE))
13+
export EXTRA_BOARD_CONFIG=$(mktemp)
14+
${BUILD_SCRIPT_PATH}/custompios_core/generate_board_config.py "${EXTRA_BOARD_CONFIG}"
15+
echo "Temp source file: ${EXTRA_BOARD_CONFIG}"
16+
1317
source ${BUILD_SCRIPT_PATH}/common.sh
1418
install_cleanup_trap
1519
1620
CUSTOM_OS_PATH=$(dirname $(realpath -s $0))
1721
18-
source ${CUSTOM_PI_OS_PATH}/config ${@}
22+
source ${CUSTOM_PI_OS_PATH}/config "${1}" "${EXTRA_BOARD_CONFIG}" ${@}
1923
${CUSTOM_PI_OS_PATH}/config_sanity
2024
2125
[ "$CONFIG_ONLY" == "yes" ] || source ${CUSTOM_OS_PATH}/custompios ${@}

‎src/common.sh‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -150,10 +150,13 @@ function unpack() {
150150
}
151151

152152
function detach_all_loopback(){
153+
image_name=$1
153154
# Cleans up mounted loopback devices from the image name
154155
# NOTE: it might need a better way to grep for the image name, its might clash with other builds
155156
for img in $(losetup | grep $1 | awk '{ print $1 }' ); do
156-
if [ -f "${img}" ] || [ -b "${img}" ]; then
157+
# test if the image name is a substring
158+
if [ "${img}" != "$(printf '%s' "${img}" | sed 's/'"${image_name}"'//g')" ] && ([ -f "${img}" ] || [ -b "${img}" ]); then
159+
echo "freeing up $img"
157160
losetup -d $img
158161
fi
159162
done

‎src/config‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@ export BUILD_VARIANT=""
77
BUILD_VARIANT="$1"
88
: ${BUILD_VARIANT:=default}
99

10+
EXTRA_BAORD_CONFIG=$2
11+
1012
export BUILD_FLAVOR=""
1113
# Disable flavor system
1214
#BUILD_FLAVOR="$1"
@@ -86,6 +88,13 @@ MODULES_LIST="${TMP//)/,}"
8688
# [ -n "$BASE_CHROOT_SCRIPT_PATH" ] || BASE_CHROOT_SCRIPT_PATH=$BASE_SCRIPT_PATH/chroot_script
8789
[ -n "$BASE_MOUNT_PATH" ] || BASE_MOUNT_PATH=$BASE_WORKSPACE/mount
8890

91+
# Import remote and submodules config
92+
if [ -f "${EXTRA_BAORD_CONFIG}" ]; then
93+
source "${EXTRA_BAORD_CONFIG}"
94+
else
95+
echo "Note: Not sourceing board config"
96+
fi
97+
8998
export REMOTE_AND_META_CONFIG="$BASE_WORKSPACE"/remote_and_meta_config
9099
# Remote modules and meta modulese go in first if they want to change standard behaviour
91100
if [ -f "${REMOTE_AND_META_CONFIG}" ]; then

‎src/custompios‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,11 @@ fi
108108
mkdir -p $BASE_WORKSPACE
109109
mkdir -p $BASE_MOUNT_PATH
110110

111+
# This is already genrated at "build" sourced in "config", but copying here mostly for debug
112+
if [ -f "${EXTRA_BAORD_CONFIG}" ]; then
113+
mv -v "${EXTRA_BAORD_CONFIG}" "${BASE_WORKSPACE}"/extra_board_config
114+
fi
115+
111116
# Clean exported artifacts from other builds
112117
rm -rf "${BASE_WORKSPACE}"/*.tar.gz
113118

@@ -121,6 +126,7 @@ pushd $BASE_WORKSPACE
121126
fi
122127
if [ ! -f "$BASE_ZIP_IMG" ] || [ "$BASE_ZIP_IMG" == "" ]; then
123128
echo "Error: could not find image: $BASE_ZIP_IMG"
129+
echo "On CustomPiOS v2 you can provide -d to download the latest image of your board automatically"
124130
exit 1
125131
fi
126132

@@ -181,7 +187,7 @@ pushd $BASE_WORKSPACE
181187
CHROOT_SCRIPT=${BASE_WORKSPACE}/chroot_script
182188
MODULES_AFTER_PATH=${BASE_WORKSPACE}/modules_after
183189
MODULES_BEFORE="${MODULES}"
184-
${CUSTOM_PI_OS_PATH}/execution_order.py "${MODULES}" "${CHROOT_SCRIPT}" "${MODULES_AFTER_PATH}" "${REMOTE_AND_META_CONFIG}"
190+
${CUSTOM_PI_OS_PATH}/custompios_core/execution_order.py "${MODULES}" "${CHROOT_SCRIPT}" "${MODULES_AFTER_PATH}" "${REMOTE_AND_META_CONFIG}"
185191
if [ -f "${REMOTE_AND_META_CONFIG}" ]; then
186192
echo "Sourcing remote and submodules config"
187193
source "${REMOTE_AND_META_CONFIG}" ${@}

‎src/custompios_core/__init__.py‎

Whitespace-only changes.

src/base_image_downloader.py renamed to src/custompios_core/base_image_downloader.py

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,13 @@
77
import hashlib
88
import shutil
99
import re
10+
from typing import Dict, Any, Optional, cast
11+
from common import get_image_config, read_images
1012
PRECENT_PROGRESS_SIZE = 5
1113

1214
class ChecksumFailException(Exception):
1315
pass
1416

15-
IMAGES_CONFIG = os.path.join(os.path.dirname(__file__), "images.yml")
1617
RETRY = 3
1718

1819
def ensure_dir(d, chmod=0o777):
@@ -26,13 +27,6 @@ def ensure_dir(d, chmod=0o777):
2627
return False
2728
return True
2829

29-
def read_images():
30-
if not os.path.isfile(IMAGES_CONFIG):
31-
raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}")
32-
with open(IMAGES_CONFIG,'r') as f:
33-
output = yaml.safe_load(f)
34-
return output
35-
3630
class DownloadProgress:
3731
last_precent: float = 0
3832
def show_progress(self, block_num, block_size, total_size):
@@ -41,8 +35,10 @@ def show_progress(self, block_num, block_size, total_size):
4135
print(f"{new_precent}%", end="\r")
4236
self.last_precent = new_precent
4337

44-
def get_file_name(headers):
45-
return re.findall("filename=(\S+)", headers["Content-Disposition"])[0]
38+
def get_file_name(headers, url):
39+
if "Content-Disposition" in headers.keys():
40+
return re.findall("filename=(\S+)", headers["Content-Disposition"])[0]
41+
return url.split('/')[-1]
4642

4743
def get_sha256(filename):
4844
sha256_hash = hashlib.sha256()
@@ -53,7 +49,7 @@ def get_sha256(filename):
5349
return file_checksum
5450
return
5551

56-
def download_image_http(board: str, dest_folder: str, redownload: bool = False):
52+
def download_image_http(board: Dict[str, Any], dest_folder: str, redownload: bool = False):
5753
url = board["url"]
5854
checksum = board["checksum"]
5955

@@ -67,7 +63,7 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
6763
# Get sha and confirm its the right image
6864
download_progress = DownloadProgress()
6965
_, headers_checksum = urllib.request.urlretrieve(checksum, temp_file_checksum, download_progress.show_progress)
70-
file_name_checksum = get_file_name(headers_checksum)
66+
file_name_checksum = get_file_name(headers_checksum, checksum)
7167

7268
checksum_data = None
7369
with open(temp_file_checksum, 'r') as f:
@@ -88,7 +84,7 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
8884
download_progress = DownloadProgress()
8985
_, headers = urllib.request.urlretrieve(url, temp_file_name, download_progress.show_progress)
9086

91-
file_name = get_file_name(headers)
87+
file_name = get_file_name(headers, url)
9288
file_checksum = get_sha256(temp_file_name)
9389
if file_checksum != online_checksum:
9490
print(f'Failed. Attempt # {r + 1}, checksum missmatch: {file_checksum} expected: {online_checksum}')
@@ -102,6 +98,7 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
10298
else:
10399
print('Error encoutered at {RETRY} attempt')
104100
print(e)
101+
exit(1)
105102
else:
106103
print(f"Success: {temp_file_name}")
107104
break
@@ -118,14 +115,25 @@ def download_image_http(board: str, dest_folder: str, redownload: bool = False):
118115
base_board = os.environ.get("BASE_BOARD", None)
119116
base_image_path = os.environ.get("BASE_IMAGE_PATH", None)
120117

121-
if base_board is not None and base_board in images["images"]:
122-
if images["images"][base_board]["type"] == "http":
123-
download_image_http(images["images"][base_board], base_image_path)
124-
elif images["images"][base_board]["type"] == "torrent":
118+
if base_image_path is None:
119+
print(f'Error: did not find image config file')
120+
exit(1)
121+
cast(str, base_image_path)
122+
123+
image_config = get_image_config()
124+
if image_config is not None:
125+
if image_config["type"] == "http":
126+
print(f"Downloading image for {base_board}")
127+
download_image_http(image_config, base_image_path)
128+
elif image_config["type"] == "torrent":
125129
print("Error: Torrent not implemented")
126130
exit(1)
127131
else:
128-
print("Error: Unsupported image download type")
132+
print(f'Error: Unsupported image download type: {image_config["type"]}')
129133
exit(1)
134+
else:
135+
print(f"Error: Image config not found for: {base_board}")
136+
exit(1)
137+
130138

131-
print("Done")
139+
print("Done")

‎src/custompios_core/common.py‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
""" Common functions between CustomPiOS python scripts"""
2+
from typing import Dict, Any, Optional, cast
3+
import yaml
4+
import os
5+
from pathlib import Path
6+
7+
def get_custompios_folder():
8+
custompios_path = os.environ.get("CUSTOM_PI_OS_PATH", None)
9+
if custompios_path is not None:
10+
return Path(custompios_path)
11+
return Path(__file__).parent.parent
12+
13+
14+
IMAGES_CONFIG = os.path.join(get_custompios_folder(), "images.yml")
15+
16+
17+
def read_images() -> Dict[str, Dict[str,str]]:
18+
if not os.path.isfile(IMAGES_CONFIG):
19+
raise Exception(f"Error: Remotes config file not found: {IMAGES_CONFIG}")
20+
with open(IMAGES_CONFIG,'r') as f:
21+
output = yaml.safe_load(f)
22+
return output
23+
24+
def get_image_config() -> Optional[Dict["str", Any]]:
25+
images = read_images()
26+
27+
base_board = os.environ.get("BASE_BOARD", None)
28+
base_image_path = os.environ.get("BASE_IMAGE_PATH", None)
29+
30+
if base_board is not None and base_board in images["images"]:
31+
return images["images"][base_board]
32+
return None

src/execution_order.py renamed to src/custompios_core/execution_order.py

File renamed without changes.

0 commit comments

Comments
 (0)