Skip to content

Commit a3bc59d

Browse files
andrasbacsaiandrasbacsai
authored
Fix Docker container race condition during upgrades (#7603)
2 parents 92326c0 + 30ac4e0 commit a3bc59d

File tree

3 files changed

+239
-27
lines changed

3 files changed

+239
-27
lines changed

‎app/Actions/Server/UpdateCoolify.php‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,6 @@ public function handle($manual_update = false)
3030
if (! $this->server) {
3131
return;
3232
}
33-
CleanupDocker::dispatch($this->server, false, false);
3433

3534
// Fetch fresh version from CDN instead of using cache
3635
try {
@@ -117,17 +116,12 @@ public function handle($manual_update = false)
117116

118117
private function update()
119118
{
120-
$helperImage = config('constants.coolify.helper_image');
121-
$latest_version = getHelperVersion();
122-
instant_remote_process(["docker pull -q {$helperImage}:{$latest_version}"], $this->server, false);
123-
124-
$image = config('constants.coolify.registry_url').'/coollabsio/coolify:'.$this->latestVersion;
125-
instant_remote_process(["docker pull -q $image"], $this->server, false);
126-
119+
$latestHelperImageVersion = getHelperVersion();
127120
$upgradeScriptUrl = config('constants.coolify.upgrade_script_url');
121+
128122
remote_process([
129123
"curl -fsSL {$upgradeScriptUrl} -o /data/coolify/source/upgrade.sh",
130-
"bash /data/coolify/source/upgrade.sh $this->latestVersion",
124+
"bash /data/coolify/source/upgrade.sh $this->latestVersion $latestHelperImageVersion",
131125
], $this->server);
132126
}
133127
}

‎other/nightly/upgrade.sh‎

Lines changed: 42 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -64,9 +64,45 @@ if [ -f /root/.docker/config.json ]; then
6464
DOCKER_CONFIG_MOUNT="-v /root/.docker/config.json:/root/.docker/config.json"
6565
fi
6666

67-
if [ -f /data/coolify/source/docker-compose.custom.yml ]; then
68-
echo "docker-compose.custom.yml detected." >>"$LOGFILE"
69-
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock ${DOCKER_CONFIG_MOUNT} --rm ${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION} bash -c "LATEST_IMAGE=${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml -f /data/coolify/source/docker-compose.custom.yml up -d --remove-orphans --wait --wait-timeout 60" >>"$LOGFILE" 2>&1
70-
else
71-
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock ${DOCKER_CONFIG_MOUNT} --rm ${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION} bash -c "LATEST_IMAGE=${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml up -d --remove-orphans --wait --wait-timeout 60" >>"$LOGFILE" 2>&1
72-
fi
67+
# Pull all required images before stopping containers
68+
# This ensures we don't take down the system if image pull fails (rate limits, network issues, etc.)
69+
echo "Pulling required Docker images..." >>"$LOGFILE"
70+
docker pull "${REGISTRY_URL:-ghcr.io}/coollabsio/coolify:${LATEST_IMAGE}" >>"$LOGFILE" 2>&1 || { echo "Failed to pull Coolify image. Aborting upgrade." >>"$LOGFILE"; exit 1; }
71+
docker pull "${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION}" >>"$LOGFILE" 2>&1 || { echo "Failed to pull Coolify helper image. Aborting upgrade." >>"$LOGFILE"; exit 1; }
72+
docker pull postgres:15-alpine >>"$LOGFILE" 2>&1 || { echo "Failed to pull PostgreSQL image. Aborting upgrade." >>"$LOGFILE"; exit 1; }
73+
docker pull redis:7-alpine >>"$LOGFILE" 2>&1 || { echo "Failed to pull Redis image. Aborting upgrade." >>"$LOGFILE"; exit 1; }
74+
# Pull realtime image - version is hardcoded in docker-compose.prod.yml, extract it or use a known version
75+
docker pull "${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-realtime:1.0.10" >>"$LOGFILE" 2>&1 || { echo "Failed to pull Coolify realtime image. Aborting upgrade." >>"$LOGFILE"; exit 1; }
76+
echo "All images pulled successfully." >>"$LOGFILE"
77+
78+
# Stop and remove existing Coolify containers to prevent conflicts
79+
# This handles both old installations (project "source") and new ones (project "coolify")
80+
# Use nohup to ensure the script continues even if SSH connection is lost
81+
echo "Starting container restart sequence (detached)..." >>"$LOGFILE"
82+
83+
nohup bash -c "
84+
LOGFILE='$LOGFILE'
85+
DOCKER_CONFIG_MOUNT='$DOCKER_CONFIG_MOUNT'
86+
REGISTRY_URL='$REGISTRY_URL'
87+
LATEST_HELPER_VERSION='$LATEST_HELPER_VERSION'
88+
LATEST_IMAGE='$LATEST_IMAGE'
89+
90+
# Stop and remove containers
91+
echo 'Stopping existing Coolify containers...' >>\"\$LOGFILE\"
92+
for container in coolify coolify-db coolify-redis coolify-realtime; do
93+
if docker ps -a --format '{{.Names}}' | grep -q \"^\${container}\$\"; then
94+
docker stop \"\$container\" >>\"\$LOGFILE\" 2>&1 || true
95+
docker rm \"\$container\" >>\"\$LOGFILE\" 2>&1 || true
96+
echo \" - Removed container: \$container\" >>\"\$LOGFILE\"
97+
fi
98+
done
99+
100+
# Start new containers
101+
if [ -f /data/coolify/source/docker-compose.custom.yml ]; then
102+
echo 'docker-compose.custom.yml detected.' >>\"\$LOGFILE\"
103+
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml -f /data/coolify/source/docker-compose.custom.yml up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1
104+
else
105+
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1
106+
fi
107+
echo 'Upgrade completed.' >>\"\$LOGFILE\"
108+
" >>"$LOGFILE" 2>&1 &

‎scripts/upgrade.sh‎

Lines changed: 194 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,23 +11,63 @@ ENV_FILE="/data/coolify/source/.env"
1111
DATE=$(date +%Y-%m-%d-%H-%M-%S)
1212
LOGFILE="/data/coolify/source/upgrade-${DATE}.log"
1313

14+
# Helper function to log with timestamp
15+
log() {
16+
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" >>"$LOGFILE"
17+
}
18+
19+
# Helper function to log section headers
20+
log_section() {
21+
echo "" >>"$LOGFILE"
22+
echo "============================================================" >>"$LOGFILE"
23+
echo "[$(date '+%Y-%m-%d %H:%M:%S')] $1" >>"$LOGFILE"
24+
echo "============================================================" >>"$LOGFILE"
25+
}
26+
27+
echo ""
28+
echo "=========================================="
29+
echo " Coolify Upgrade - ${DATE}"
30+
echo "=========================================="
31+
echo ""
32+
33+
# Initialize log file with header
34+
echo "============================================================" >>"$LOGFILE"
35+
echo "Coolify Upgrade Log" >>"$LOGFILE"
36+
echo "Started: $(date '+%Y-%m-%d %H:%M:%S')" >>"$LOGFILE"
37+
echo "Target Version: ${LATEST_IMAGE}" >>"$LOGFILE"
38+
echo "Helper Version: ${LATEST_HELPER_VERSION}" >>"$LOGFILE"
39+
echo "Registry URL: ${REGISTRY_URL}" >>"$LOGFILE"
40+
echo "============================================================" >>"$LOGFILE"
41+
42+
log_section "Step 1/6: Downloading configuration files"
43+
echo "1/6 Downloading latest configuration files..."
44+
log "Downloading docker-compose.yml from ${CDN}/docker-compose.yml"
1445
curl -fsSL -L $CDN/docker-compose.yml -o /data/coolify/source/docker-compose.yml
46+
log "Downloading docker-compose.prod.yml from ${CDN}/docker-compose.prod.yml"
1547
curl -fsSL -L $CDN/docker-compose.prod.yml -o /data/coolify/source/docker-compose.prod.yml
48+
log "Downloading .env.production from ${CDN}/.env.production"
1649
curl -fsSL -L $CDN/.env.production -o /data/coolify/source/.env.production
50+
log "Configuration files downloaded successfully"
51+
echo " Done."
1752

1853
# Backup existing .env file before making any changes
1954
if [ "$SKIP_BACKUP" != "true" ]; then
2055
if [ -f "$ENV_FILE" ]; then
21-
echo "Creating backup of existing .env file to .env-$DATE" >>"$LOGFILE"
56+
echo " Creating backup of .env file..."
57+
log "Creating backup of .env file to .env-$DATE"
2258
cp "$ENV_FILE" "$ENV_FILE-$DATE"
59+
log "Backup created: ${ENV_FILE}-${DATE}"
2360
else
24-
echo "No existing .env file found to backup" >>"$LOGFILE"
61+
log "WARNING: No existing .env file found to backup"
2562
fi
2663
fi
2764

28-
echo "Merging .env.production values into .env" >>"$LOGFILE"
65+
log_section "Step 2/6: Updating environment configuration"
66+
echo ""
67+
echo "2/6 Updating environment configuration..."
68+
log "Merging .env.production values into .env"
2969
awk -F '=' '!seen[$1]++' "$ENV_FILE" /data/coolify/source/.env.production > "$ENV_FILE.tmp" && mv "$ENV_FILE.tmp" "$ENV_FILE"
30-
echo ".env file merged successfully" >>"$LOGFILE"
70+
log "Environment file merged successfully"
3171

3272
update_env_var() {
3373
local key="$1"
@@ -36,37 +76,179 @@ update_env_var() {
3676
# If variable "key=" exists but has no value, update the value of the existing line
3777
if grep -q "^${key}=$" "$ENV_FILE"; then
3878
sed -i "s|^${key}=$|${key}=${value}|" "$ENV_FILE"
39-
echo " - Updated value of ${key} as the current value was empty" >>"$LOGFILE"
79+
log "Updated ${key} (was empty)"
4080
# If variable "key=" doesn't exist, append it to the file with value
4181
elif ! grep -q "^${key}=" "$ENV_FILE"; then
4282
printf '%s=%s\n' "$key" "$value" >>"$ENV_FILE"
43-
echo " - Added ${key} with default value as the variable was missing" >>"$LOGFILE"
83+
log "Added ${key} (was missing)"
4484
fi
4585
}
4686

47-
echo "Checking and updating environment variables if necessary..." >>"$LOGFILE"
87+
log "Checking environment variables..."
4888
update_env_var "PUSHER_APP_ID" "$(openssl rand -hex 32)"
4989
update_env_var "PUSHER_APP_KEY" "$(openssl rand -hex 32)"
5090
update_env_var "PUSHER_APP_SECRET" "$(openssl rand -hex 32)"
91+
log "Environment variables check complete"
92+
echo " Done."
5193

5294
# Make sure coolify network exists
5395
# It is created when starting Coolify with docker compose
96+
log "Checking Docker network 'coolify'..."
5497
if ! docker network inspect coolify >/dev/null 2>&1; then
98+
log "Network 'coolify' does not exist, creating..."
5599
if ! docker network create --attachable --ipv6 coolify 2>/dev/null; then
56-
echo "Failed to create coolify network with ipv6. Trying without ipv6..."
100+
log "Failed to create network with IPv6, trying without IPv6..."
57101
docker network create --attachable coolify 2>/dev/null
102+
log "Network 'coolify' created without IPv6"
103+
else
104+
log "Network 'coolify' created with IPv6 support"
58105
fi
106+
else
107+
log "Network 'coolify' already exists"
59108
fi
60109

61110
# Check if Docker config file exists
62111
DOCKER_CONFIG_MOUNT=""
63112
if [ -f /root/.docker/config.json ]; then
64113
DOCKER_CONFIG_MOUNT="-v /root/.docker/config.json:/root/.docker/config.json"
114+
log "Docker config mount enabled: /root/.docker/config.json"
65115
fi
66116

67-
if [ -f /data/coolify/source/docker-compose.custom.yml ]; then
68-
echo "docker-compose.custom.yml detected." >>"$LOGFILE"
69-
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock ${DOCKER_CONFIG_MOUNT} --rm ${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION} bash -c "LATEST_IMAGE=${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml -f /data/coolify/source/docker-compose.custom.yml up -d --remove-orphans --wait --wait-timeout 60" >>"$LOGFILE" 2>&1
117+
log_section "Step 3/6: Pulling Docker images"
118+
echo ""
119+
echo "3/6 Pulling Docker images..."
120+
echo " This may take a few minutes depending on your connection."
121+
122+
echo " - Pulling Coolify image..."
123+
log "Pulling image: ${REGISTRY_URL:-ghcr.io}/coollabsio/coolify:${LATEST_IMAGE}"
124+
if docker pull "${REGISTRY_URL:-ghcr.io}/coollabsio/coolify:${LATEST_IMAGE}" >>"$LOGFILE" 2>&1; then
125+
log "Successfully pulled Coolify image"
70126
else
71-
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock ${DOCKER_CONFIG_MOUNT} --rm ${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION} bash -c "LATEST_IMAGE=${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml up -d --remove-orphans --wait --wait-timeout 60" >>"$LOGFILE" 2>&1
127+
log "ERROR: Failed to pull Coolify image"
128+
echo " ERROR: Failed to pull Coolify image. Aborting upgrade."
129+
exit 1
72130
fi
131+
132+
echo " - Pulling Coolify helper image..."
133+
log "Pulling image: ${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION}"
134+
if docker pull "${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:${LATEST_HELPER_VERSION}" >>"$LOGFILE" 2>&1; then
135+
log "Successfully pulled Coolify helper image"
136+
else
137+
log "ERROR: Failed to pull Coolify helper image"
138+
echo " ERROR: Failed to pull helper image. Aborting upgrade."
139+
exit 1
140+
fi
141+
142+
echo " - Pulling PostgreSQL image..."
143+
log "Pulling image: postgres:15-alpine"
144+
if docker pull postgres:15-alpine >>"$LOGFILE" 2>&1; then
145+
log "Successfully pulled PostgreSQL image"
146+
else
147+
log "ERROR: Failed to pull PostgreSQL image"
148+
echo " ERROR: Failed to pull PostgreSQL image. Aborting upgrade."
149+
exit 1
150+
fi
151+
152+
echo " - Pulling Redis image..."
153+
log "Pulling image: redis:7-alpine"
154+
if docker pull redis:7-alpine >>"$LOGFILE" 2>&1; then
155+
log "Successfully pulled Redis image"
156+
else
157+
log "ERROR: Failed to pull Redis image"
158+
echo " ERROR: Failed to pull Redis image. Aborting upgrade."
159+
exit 1
160+
fi
161+
162+
echo " - Pulling Coolify realtime image..."
163+
log "Pulling image: ${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-realtime:1.0.10"
164+
if docker pull "${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-realtime:1.0.10" >>"$LOGFILE" 2>&1; then
165+
log "Successfully pulled Coolify realtime image"
166+
else
167+
log "ERROR: Failed to pull Coolify realtime image"
168+
echo " ERROR: Failed to pull realtime image. Aborting upgrade."
169+
exit 1
170+
fi
171+
172+
log "All images pulled successfully"
173+
echo " All images pulled successfully."
174+
175+
log_section "Step 4/6: Stopping and restarting containers"
176+
echo ""
177+
echo "4/6 Stopping containers and starting new ones..."
178+
echo " This step will restart all Coolify containers."
179+
echo " Check the log file for details: ${LOGFILE}"
180+
181+
# From this point forward, we need to ensure the script continues even if
182+
# the SSH connection is lost (which happens when coolify container stops)
183+
# We use a subshell with nohup to ensure completion
184+
log "Starting container restart sequence (detached)..."
185+
186+
nohup bash -c "
187+
LOGFILE='$LOGFILE'
188+
DOCKER_CONFIG_MOUNT='$DOCKER_CONFIG_MOUNT'
189+
REGISTRY_URL='$REGISTRY_URL'
190+
LATEST_HELPER_VERSION='$LATEST_HELPER_VERSION'
191+
LATEST_IMAGE='$LATEST_IMAGE'
192+
193+
log() {
194+
echo \"[\$(date '+%Y-%m-%d %H:%M:%S')] \$1\" >>\"\$LOGFILE\"
195+
}
196+
197+
# Stop and remove containers
198+
for container in coolify coolify-db coolify-redis coolify-realtime; do
199+
if docker ps -a --format '{{.Names}}' | grep -q \"^\${container}\$\"; then
200+
log \"Stopping container: \${container}\"
201+
docker stop \"\$container\" >>\"\$LOGFILE\" 2>&1 || true
202+
log \"Removing container: \${container}\"
203+
docker rm \"\$container\" >>\"\$LOGFILE\" 2>&1 || true
204+
log \"Container \${container} stopped and removed\"
205+
else
206+
log \"Container \${container} not found (skipping)\"
207+
fi
208+
done
209+
log \"Container cleanup complete\"
210+
211+
# Start new containers
212+
echo '' >>\"\$LOGFILE\"
213+
echo '============================================================' >>\"\$LOGFILE\"
214+
log 'Step 5/6: Starting new containers'
215+
echo '============================================================' >>\"\$LOGFILE\"
216+
217+
if [ -f /data/coolify/source/docker-compose.custom.yml ]; then
218+
log 'Using custom docker-compose.yml'
219+
log 'Running docker compose up with custom configuration...'
220+
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} --rm \${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml -f /data/coolify/source/docker-compose.custom.yml up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1
221+
else
222+
log 'Using standard docker-compose configuration'
223+
log 'Running docker compose up...'
224+
docker run -v /data/coolify/source:/data/coolify/source -v /var/run/docker.sock:/var/run/docker.sock \${DOCKER_CONFIG_MOUNT} \${REGISTRY_URL:-ghcr.io}/coollabsio/coolify-helper:\${LATEST_HELPER_VERSION} bash -c \"LATEST_IMAGE=\${LATEST_IMAGE} docker compose --project-name coolify --env-file /data/coolify/source/.env -f /data/coolify/source/docker-compose.yml -f /data/coolify/source/docker-compose.prod.yml up -d --remove-orphans --wait --wait-timeout 60\" >>\"\$LOGFILE\" 2>&1
225+
fi
226+
log 'Docker compose up completed'
227+
228+
# Final log entry
229+
echo '' >>\"\$LOGFILE\"
230+
echo '============================================================' >>\"\$LOGFILE\"
231+
log 'Step 6/6: Upgrade complete'
232+
echo '============================================================' >>\"\$LOGFILE\"
233+
log 'Coolify upgrade completed successfully'
234+
log \"Version: \${LATEST_IMAGE}\"
235+
echo '' >>\"\$LOGFILE\"
236+
echo '============================================================' >>\"\$LOGFILE\"
237+
echo \"Upgrade completed: \$(date '+%Y-%m-%d %H:%M:%S')\" >>\"\$LOGFILE\"
238+
echo '============================================================' >>\"\$LOGFILE\"
239+
" >>"$LOGFILE" 2>&1 &
240+
241+
# Give the background process a moment to start
242+
sleep 2
243+
log "Container restart sequence started in background (PID: $!)"
244+
echo ""
245+
echo "5/6 Containers are being restarted in the background..."
246+
echo "6/6 Upgrade process initiated!"
247+
echo ""
248+
echo "=========================================="
249+
echo " Coolify upgrade to ${LATEST_IMAGE} in progress"
250+
echo "=========================================="
251+
echo ""
252+
echo " The upgrade will continue in the background."
253+
echo " Coolify will be available again shortly."
254+
echo " Log file: ${LOGFILE}"

0 commit comments

Comments
 (0)