Skip to content

Commit 6d2d2e9

Browse files
igorpecovnikigorpecovnik
committed
Fix repository management script for parallel execution
- Remove redundant run_cmd function, use run_aptly consistently - Include EOS releases in repository processing (remove -ve 'eos' filter) - Fix showall command to auto-discover repos from isolated databases - Fix source file preservation in parallel mode (keep sources for workers) - Fix snapshot management to handle published snapshots correctly - Fix drop_unsupported_releases to only drop unsupported, not all repos - Remove isolated DB cleanup that caused data loss on each run - Remove inappropriate sudo usage from html and date commands - Enable FORCE_PUBLISH by default for better workflow - Improve common snapshot creation in isolated mode - Add isolated database support to showall function Signed-off-by: Igor Pecovnik <igor@armbian.com>
1 parent 4648add commit 6d2d2e9

File tree

1 file changed

+118
-65
lines changed

1 file changed

+118
-65
lines changed

‎tools/repository/repo.sh‎

Lines changed: 118 additions & 65 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ DRY_RUN=false # Full dry-run: don't make any repository changes
55
KEEP_SOURCES=false # Keep source packages when adding to repo (don't delete)
66
SINGLE_RELEASE="" # Process only a single release (for GitHub Actions parallel workflow)
77
FORCE_ADD=false # Force re-adding packages even if they already exist in repo
8-
FORCE_PUBLISH=false # Force publishing even when no packages to add
8+
FORCE_PUBLISH=true # Force publishing even when no packages to add
99

1010
# Logging function - uses syslog, view logs with: journalctl -t repo-management -f
1111
# Arguments:
@@ -14,24 +14,6 @@ log() {
1414
logger -t repo-management "$*"
1515
}
1616

17-
# Execute a command, respecting dry-run mode
18-
# In dry-run mode, logs what would be executed without actually running it
19-
# Arguments:
20-
# $* - Command to execute
21-
# Returns:
22-
# Command exit status (0 in dry-run mode)
23-
run_cmd() {
24-
local cmd="$*"
25-
if [[ "$DRY_RUN" == true ]]; then
26-
log "[DRY-RUN] Would execute: $cmd"
27-
return 0
28-
else
29-
log "Executing: $cmd"
30-
eval "$cmd"
31-
return $?
32-
fi
33-
}
34-
3517
# Execute aptly command and check for errors
3618
# Exits with status 1 if the command fails (unless in dry-run mode)
3719
# Arguments:
@@ -65,7 +47,7 @@ drop_unsupported_releases() {
6547
supported_releases=()
6648
else
6749
log "Cleanup: dropping unsupported releases"
68-
supported_releases=($(grep -rw config/distributions/*/support -ve 'eos' | cut -d"/" -f3))
50+
supported_releases=($(grep -rw config/distributions/*/support | cut -d"/" -f3))
6951
fi
7052

7153
# Get currently published repositories
@@ -82,25 +64,74 @@ drop_unsupported_releases() {
8264

8365
# Drop the identified repositories
8466
for repo in "${repos_to_drop[@]}"; do
85-
run_cmd aptly publish drop -config="${CONFIG}" "${repo}"
67+
run_aptly publish drop -config="${CONFIG}" "${repo}"
8668
done
8769
}
8870
# Display contents of all repositories
8971
# Shows packages in the common repository and release-specific repositories (utils, desktop)
90-
# Uses global DISTROS array for iteration
72+
# In single-release mode, shows content from isolated database
73+
# Otherwise, shows content from main database and any existing isolated databases
74+
# Uses global DISTROS array for iteration, or discovers repos automatically if DISTROS is empty
9175
showall() {
9276
echo "Displaying common repository contents"
9377
aptly repo show -with-packages -config="${CONFIG}" common 2>/dev/null | tail -n +7
9478

95-
for release in "${DISTROS[@]}"; do
96-
# Only show if the repo exists
97-
if aptly repo show -config="${CONFIG}" "${release}-utils" &>/dev/null; then
79+
# If DISTROS array is empty, discover repos from the database
80+
local releases_to_show=("${DISTROS[@]}")
81+
if [[ ${#DISTROS[@]} -eq 0 ]]; then
82+
# First, discover releases from isolated databases
83+
local all_repos=()
84+
if [[ -d "$output" ]]; then
85+
for isolated_dir in "$output"/aptly-isolated-*; do
86+
if [[ -d "$isolated_dir" ]]; then
87+
local release_name=$(basename "$isolated_dir" | sed 's/aptly-isolated-//')
88+
all_repos+=("$release_name")
89+
fi
90+
done
91+
fi
92+
# Also get repos from main database (for non-isolated repos)
93+
local main_repos
94+
main_repos=($(aptly repo list -config="${CONFIG}" -raw 2>/dev/null | awk '{print $NF}' | grep -E '^.+-(utils|desktop)$' | sed 's/-(utils|desktop)$//' | sort -u))
95+
# Merge and deduplicate
96+
all_repos+=("${main_repos[@]}")
97+
releases_to_show=($(echo "${all_repos[@]}" | tr ' ' '\n' | sort -u))
98+
fi
99+
100+
for release in "${releases_to_show[@]}"; do
101+
# In single-release mode, only show that specific release from the isolated database
102+
if [[ -n "$SINGLE_RELEASE" ]]; then
103+
if [[ "$release" != "$SINGLE_RELEASE" ]]; then
104+
continue
105+
fi
106+
fi
107+
108+
# Check if there's an isolated database for this release
109+
local isolated_db="${output}/aptly-isolated-${release}"
110+
local show_config="$CONFIG"
111+
112+
if [[ -d "$isolated_db" ]]; then
113+
# Create temporary config for the isolated database
114+
local temp_config
115+
temp_config="$(mktemp)"
116+
sed 's|"rootDir": ".*"|"rootDir": "'$isolated_db'"|g' tools/repository/aptly.conf > "$temp_config"
117+
show_config="$temp_config"
118+
fi
119+
120+
# Show utils repo if it exists
121+
if aptly repo show -config="${show_config}" "${release}-utils" &>/dev/null; then
98122
echo "Displaying repository contents for $release-utils"
99-
aptly repo show -with-packages -config="${CONFIG}" "${release}-utils" | tail -n +7
123+
aptly repo show -with-packages -config="${show_config}" "${release}-utils" | tail -n +7
100124
fi
101-
if aptly repo show -config="${CONFIG}" "${release}-desktop" &>/dev/null; then
125+
126+
# Show desktop repo if it exists
127+
if aptly repo show -config="${show_config}" "${release}-desktop" &>/dev/null; then
102128
echo "Displaying repository contents for $release-desktop"
103-
aptly repo show -with-packages -config="${CONFIG}" "${release}-desktop" | tail -n +7
129+
aptly repo show -with-packages -config="${show_config}" "${release}-desktop" | tail -n +7
130+
fi
131+
132+
# Clean up temp config if we created one
133+
if [[ -n "$temp_config" && -f "$temp_config" ]]; then
134+
rm -f "$temp_config"
104135
fi
105136
done
106137
}
@@ -205,8 +236,9 @@ adding_packages() {
205236
# Determine whether to remove source files after adding to repo
206237
# KEEP_SOURCES mode preserves source packages
207238
# DRY_RUN mode also preserves sources (and skips all repo modifications)
239+
# SINGLE_RELEASE mode preserves sources so parallel workers don't delete files needed by other workers
208240
local remove_flag="-remove-files"
209-
if [[ "$KEEP_SOURCES" == true ]] || [[ "$DRY_RUN" == true ]]; then
241+
if [[ "$KEEP_SOURCES" == true ]] || [[ "$DRY_RUN" == true ]] || [[ -n "$SINGLE_RELEASE" ]]; then
210242
remove_flag=""
211243
fi
212244

@@ -239,13 +271,24 @@ update_main() {
239271
# Add packages from main folder
240272
adding_packages "common" "" "main" "$input_folder"
241273

242-
# Drop old snapshot
274+
# Drop old snapshot if it exists and is not published
243275
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "common") ]]; then
244-
run_aptly -config="${CONFIG}" snapshot drop common | logger -t repo-management >/dev/null
276+
# Check if snapshot is published
277+
if ! aptly publish list -config="${CONFIG}" 2>/dev/null | grep -q "common"; then
278+
run_aptly -config="${CONFIG}" snapshot drop common | logger -t repo-management >/dev/null
279+
else
280+
log "WARNING: common snapshot is published, cannot drop. Packages added to repo but snapshot not updated."
281+
log "Run 'update' command to update all releases with new packages."
282+
return 0
283+
fi
245284
fi
246285

247-
# Create new snapshot
248-
run_aptly -config="${CONFIG}" snapshot create common from repo common | logger -t repo-management >/dev/null
286+
# Create new snapshot if it doesn't exist or was dropped
287+
if [[ -z $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "common") ]]; then
288+
run_aptly -config="${CONFIG}" snapshot create common from repo common | logger -t repo-management >/dev/null
289+
else
290+
log "common snapshot already exists, skipping creation"
291+
fi
249292

250293
log "Common component built successfully"
251294
}
@@ -267,18 +310,26 @@ process_release() {
267310
log "Processing release: $release"
268311

269312
# In isolated mode (SINGLE_RELEASE), ensure common snapshot exists
270-
# It should have been created by 'update-main' command, but if not, create empty common
271-
if [[ -n "$SINGLE_RELEASE" && -z $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "common") ]]; then
272-
log "WARNING: Common snapshot not found. Creating empty common snapshot."
273-
log "Please run 'update-main' command first to populate common packages."
274-
275-
# Create empty common repo
313+
# It should have been created by 'update-main' command, but if not, create it from input packages
314+
if [[ -n "$SINGLE_RELEASE" ]]; then
315+
# Create common repo if it doesn't exist
276316
if [[ -z $(aptly repo list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep common) ]]; then
277-
aptly repo create -config="${CONFIG}" -distribution="common" -component="main" -comment="Armbian common packages" "common" | logger -t repo-management >/dev/null
317+
run_aptly repo create -config="${CONFIG}" -distribution="common" -component="main" -comment="Armbian common packages" "common" | logger -t repo-management >/dev/null
278318
fi
279319

280-
# Create snapshot (will be empty until update-main is run)
281-
aptly -config="${CONFIG}" snapshot create common from repo common | logger -t repo-management >/dev/null
320+
# Add packages from main input folder to common repo
321+
# This ensures each isolated worker has the common packages
322+
log "Populating common repo from input folder: $input_folder"
323+
adding_packages "common" "" "main" "$input_folder"
324+
325+
# Drop old common snapshot if it exists (in isolated DB, snapshots aren't published yet)
326+
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "common") ]]; then
327+
run_aptly -config="${CONFIG}" snapshot drop common | logger -t repo-management >/dev/null
328+
fi
329+
330+
# Create snapshot with packages
331+
run_aptly -config="${CONFIG}" snapshot create common from repo common | logger -t repo-management >/dev/null
332+
log "Created common snapshot with packages for isolated mode"
282333
fi
283334

284335
# Create release-specific repositories if they don't exist
@@ -305,13 +356,11 @@ process_release() {
305356

306357
log "Package counts for $release: utils=$utils_count, desktop=$desktop_count"
307358

308-
# If no packages in either repo and not previously published, skip publishing
309-
# Unless FORCE_PUBLISH is enabled
359+
# Always publish - even if no release-specific packages, we still need to publish common/main
360+
# Check if this release was previously published for logging
310361
if [[ "$utils_count" -eq 0 && "$desktop_count" -eq 0 && "$FORCE_PUBLISH" != true ]]; then
311-
# Check if this release was previously published
312362
if ! aptly publish list -config="${CONFIG}" 2>/dev/null | grep -q "^\[${release}\]"; then
313-
log "No packages to publish for $release and not previously published. Skipping."
314-
return 0
363+
log "No release-specific packages for $release. Publishing common/main component only."
315364
else
316365
log "No new packages but $release was previously published. Will publish with common only."
317366
fi
@@ -321,12 +370,19 @@ process_release() {
321370
log "Force publish enabled: will publish even with no packages"
322371
fi
323372

324-
# Drop old snapshots if they exist
325-
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-utils") ]]; then
326-
run_aptly -config="${CONFIG}" snapshot drop ${release}-utils | logger -t repo-management 2>/dev/null
373+
# Drop old snapshots if we have new packages to add OR if FORCE_PUBLISH is enabled
374+
# This ensures fresh snapshots are created for force-publish scenarios
375+
if [[ "$utils_count" -gt 0 || "$FORCE_PUBLISH" == true ]]; then
376+
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-utils") ]]; then
377+
log "Dropping existing ${release}-utils snapshot"
378+
run_aptly -config="${CONFIG}" snapshot drop ${release}-utils | logger -t repo-management 2>/dev/null
379+
fi
327380
fi
328-
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-desktop") ]]; then
329-
run_aptly -config="${CONFIG}" snapshot drop ${release}-desktop | logger -t repo-management 2>/dev/null
381+
if [[ "$desktop_count" -gt 0 || "$FORCE_PUBLISH" == true ]]; then
382+
if [[ -n $(aptly snapshot list -config="${CONFIG}" -raw | awk '{print $(NF)}' | grep "${release}-desktop") ]]; then
383+
log "Dropping existing ${release}-desktop snapshot"
384+
run_aptly -config="${CONFIG}" snapshot drop ${release}-desktop | logger -t repo-management 2>/dev/null
385+
fi
330386
fi
331387

332388
# Create snapshots only for repos that have packages
@@ -426,11 +482,14 @@ process_release() {
426482
-label="Armbian" \
427483
-config="${CONFIG}" \
428484
-component="$component_list" \
429-
-distribution="${release}" snapshot $snapshot_list > /dev/null
485+
-distribution="${release}" snapshot $snapshot_list
430486

431487
# If using isolated DB, copy published files to shared output location FIRST
488+
log "Isolated mode check: SINGLE_RELEASE='$SINGLE_RELEASE' publish_dir='$publish_dir' output_folder='$output_folder'"
432489
if [[ -n "$SINGLE_RELEASE" && "$publish_dir" != "$output_folder" ]]; then
433490
log "Copying published files from isolated DB to shared output"
491+
log "Source: ${publish_dir}/public"
492+
log "Destination: ${output_folder}/public"
434493
if [[ -d "${publish_dir}/public" ]]; then
435494
mkdir -p "${output_folder}/public"
436495
# Use rsync to copy published repo files to shared location
@@ -684,7 +743,7 @@ merge_repos() {
684743
log "Copied GPG key to repository"
685744

686745
# Write repository sync control file
687-
sudo date +%s > ${output_folder}/public/control
746+
date +%s > ${output_folder}/public/control
688747
log "Updated repository control file"
689748

690749
# Display repository contents
@@ -755,7 +814,7 @@ repo-manipulate() {
755814
echo "<thead><tr><td colspan=3><h2>$release</h2></tr><tr><th>Main</th><th>Utils</th><th>Desktop</th></tr></thead>"
756815
echo "<tbody><tr><td width=33% valign=top>"
757816
aptly repo show -with-packages -config="${CONFIG}" "${release}-utils" | tail -n +7 | sed 's/.*/&<br>/'
758-
echo "</td><td width=33% valign=top>" | sudo tee -a ${filename}
817+
echo "</td><td width=33% valign=top>"
759818
aptly repo show -with-packages -config="${CONFIG}" "${release}-desktop" | tail -n +7 | sed 's/.*/&<br>/'
760819
echo "</td></tr></tbody>"
761820
done
@@ -822,8 +881,8 @@ repo-manipulate() {
822881
;;
823882

824883
update)
825-
# remove old releases from publishing
826-
drop_unsupported_releases "all"
884+
# remove old releases from publishing (only drops unsupported releases, not all)
885+
drop_unsupported_releases ""
827886
publishing "$1" "$2" "$3" "$4" "$5"
828887
# Only use signing function for non-single-release mode
829888
# In single-release mode, workers already signed their components
@@ -851,7 +910,7 @@ input="output/debs-beta"
851910
output="output/repository"
852911
command="show"
853912
if [[ -d "config/distributions" ]]; then
854-
releases=$(grep -rw config/distributions/*/support -ve 'eos' 2>/dev/null | cut -d"/" -f3 | xargs | sed -e 's/ /,/g')
913+
releases=$(grep -rw config/distributions/*/support 2>/dev/null | cut -d"/" -f3 | xargs | sed -e 's/ /,/g')
855914
if [[ -z "$releases" ]]; then
856915
log "WARNING: No releases found in config/distributions"
857916
fi
@@ -1000,13 +1059,7 @@ if [[ -n "$SINGLE_RELEASE" ]]; then
10001059
# Create isolated aptly directory for this release
10011060
IsolatedRootDir="${output}/aptly-isolated-${SINGLE_RELEASE}"
10021061

1003-
# Clean up isolated DB from previous runs to ensure fresh state
1004-
# This prevents database/pool desync and "no such file" errors
1005-
if [[ -d "$IsolatedRootDir" ]]; then
1006-
log "Cleaning up isolated DB from previous run: $IsolatedRootDir"
1007-
rm -rf "${IsolatedRootDir}"
1008-
fi
1009-
1062+
# Create the isolated directory if it doesn't exist
10101063
if ! mkdir -p "$IsolatedRootDir"; then
10111064
log "ERROR: mkdir $IsolatedRootDir: permission denied"
10121065
exit 1

0 commit comments

Comments
 (0)