better timeouts

This commit is contained in:
2025-06-23 09:35:15 +02:00
parent 0da5d9305d
commit 0bbdf0eaab
3 changed files with 96 additions and 54 deletions

View File

@@ -350,7 +350,7 @@ def init_routes(main_bp):
try: try:
# Construct the health check URL # Construct the health check URL
health_url = f"{instance.main_url.rstrip('/')}/health" health_url = f"{instance.main_url.rstrip('/')}/health"
response = requests.get(health_url, timeout=5) response = requests.get(health_url, timeout=30) # Increased timeout to 30 seconds
if response.status_code == 200: if response.status_code == 200:
data = response.json() data = response.json()
@@ -397,7 +397,7 @@ def init_routes(main_bp):
deployed_commit = None deployed_commit = None
try: try:
version_url = f"{instance.main_url.rstrip('/')}/api/version" version_url = f"{instance.main_url.rstrip('/')}/api/version"
resp = requests.get(version_url, timeout=5) resp = requests.get(version_url, timeout=30) # Increased timeout to 30 seconds
if resp.status_code == 200: if resp.status_code == 200:
version_data = resp.json() version_data = resp.json()
deployed_version = version_data.get('version', 'unknown') deployed_version = version_data.get('version', 'unknown')
@@ -419,7 +419,7 @@ def init_routes(main_bp):
headers = {'Accept': 'application/json', 'Authorization': f'token {gitea_token}'} headers = {'Accept': 'application/json', 'Authorization': f'token {gitea_token}'}
# Gitea API: /api/v1/repos/{owner}/{repo}/commits/{branch} # Gitea API: /api/v1/repos/{owner}/{repo}/commits/{branch}
commit_url = f"{gitea_url}/api/v1/repos/{gitea_repo}/commits/{deployed_branch}" commit_url = f"{gitea_url}/api/v1/repos/{gitea_repo}/commits/{deployed_branch}"
commit_resp = requests.get(commit_url, headers=headers, timeout=5) commit_resp = requests.get(commit_url, headers=headers, timeout=30) # Increased timeout to 30 seconds
if commit_resp.status_code == 200: if commit_resp.status_code == 200:
latest_version = commit_resp.json().get('sha') latest_version = commit_resp.json().get('sha')
except Exception as e: except Exception as e:

View File

@@ -571,40 +571,26 @@ async function startLaunch(data) {
saveDataStep.classList.add('completed'); saveDataStep.classList.add('completed');
saveDataStep.querySelector('.step-status').textContent = 'Successfully saved instance data'; saveDataStep.querySelector('.step-status').textContent = 'Successfully saved instance data';
// Add instance details
const instanceDetails = document.createElement('div');
instanceDetails.className = 'mt-3';
instanceDetails.innerHTML = `
<div class="card">
<div class="card-body">
<h6 class="card-title mb-3">Instance Information</h6>
<div class="table-responsive">
<table class="table table-sm">
<thead>
<tr>
<th>Property</th>
<th>Value</th>
</tr>
</thead>
<tbody>
<tr>
<td>Internal Port</td>
<td>${data.port}</td>
</tr>
<tr>
<td>Domains</td>
<td><a href="https://${data.webAddresses.join(', ')}" target="_blank">${data.webAddresses.join(', ')}</a></td>
</tr>
</tbody>
</table>
</div>
</div>
</div>
`;
saveDataStep.querySelector('.step-content').appendChild(instanceDetails);
// After saving instance data, add the health check step // After saving instance data, add the health check step
await updateStep(11, 'Health Check', 'Verifying instance health...'); await updateStep(11, 'Health Check', 'Verifying instance health...');
// Add a progress indicator
const healthStepElement = document.querySelectorAll('.step-item')[10];
const progressDiv = document.createElement('div');
progressDiv.className = 'mt-2';
progressDiv.innerHTML = `
<div class="progress" style="height: 20px;">
<div class="progress-bar progress-bar-striped progress-bar-animated"
role="progressbar"
style="width: 0%"
id="healthProgress">
0%
</div>
</div>
<small class="text-muted" id="healthProgressText">Starting health check...</small>
`;
healthStepElement.querySelector('.step-content').appendChild(progressDiv);
const healthResult = await checkInstanceHealth(`https://${data.webAddresses[0]}`); const healthResult = await checkInstanceHealth(`https://${data.webAddresses[0]}`);
if (!healthResult.success) { if (!healthResult.success) {
@@ -612,7 +598,7 @@ async function startLaunch(data) {
} }
// Add a retry button if health check fails // Add a retry button if health check fails
const healthStep = document.querySelectorAll('.step-item')[10]; const healthStepElement2 = document.querySelectorAll('.step-item')[10];
if (!healthResult.success) { if (!healthResult.success) {
const retryButton = document.createElement('button'); const retryButton = document.createElement('button');
retryButton.className = 'btn btn-sm btn-warning mt-2'; retryButton.className = 'btn btn-sm btn-warning mt-2';
@@ -622,15 +608,15 @@ async function startLaunch(data) {
retryButton.innerHTML = '<i class="fas fa-spinner fa-spin me-1"></i> Checking...'; retryButton.innerHTML = '<i class="fas fa-spinner fa-spin me-1"></i> Checking...';
const retryResult = await checkInstanceHealth(`https://${data.webAddresses[0]}`); const retryResult = await checkInstanceHealth(`https://${data.webAddresses[0]}`);
if (retryResult.success) { if (retryResult.success) {
healthStep.classList.remove('failed'); healthStepElement2.classList.remove('failed');
healthStep.classList.add('completed'); healthStepElement2.classList.add('completed');
retryButton.remove(); retryButton.remove();
} else { } else {
retryButton.disabled = false; retryButton.disabled = false;
retryButton.innerHTML = '<i class="fas fa-sync-alt me-1"></i> Retry Health Check'; retryButton.innerHTML = '<i class="fas fa-sync-alt me-1"></i> Retry Health Check';
} }
}; };
healthStep.querySelector('.step-content').appendChild(retryButton); healthStepElement2.querySelector('.step-content').appendChild(retryButton);
} }
// After health check, add authentication step // After health check, add authentication step
@@ -2242,11 +2228,19 @@ async function saveInstanceData(instanceData) {
} }
async function checkInstanceHealth(instanceUrl) { async function checkInstanceHealth(instanceUrl) {
const maxRetries = 5; const maxRetries = 120; // 120 retries * 5 seconds = 10 minutes total
const baseDelay = 5000; // 5 seconds base delay
let currentAttempt = 1; let currentAttempt = 1;
const startTime = Date.now();
const maxTotalTime = 10 * 60 * 1000; // 10 minutes in milliseconds
while (currentAttempt <= maxRetries) { while (currentAttempt <= maxRetries) {
try { try {
// Check if we've exceeded the total timeout
if (Date.now() - startTime > maxTotalTime) {
throw new Error('Health check timeout: 10 minutes exceeded');
}
// First get the instance ID from the database // First get the instance ID from the database
const response = await fetch('/instances'); const response = await fetch('/instances');
const text = await response.text(); const text = await response.text();
@@ -2280,16 +2274,31 @@ async function checkInstanceHealth(instanceUrl) {
const data = await statusResponse.json(); const data = await statusResponse.json();
// Update the health check step // Update the health check step
const healthStep = document.querySelectorAll('.step-item')[8]; // Adjust index based on your steps const healthStepElement = document.querySelectorAll('.step-item')[10]; // Adjust index based on your steps
healthStep.classList.remove('active'); healthStepElement.classList.remove('active');
healthStep.classList.add('completed'); healthStepElement.classList.add('completed');
const statusText = healthStep.querySelector('.step-status'); const statusText = healthStepElement.querySelector('.step-status');
if (data.status === 'active') { if (data.status === 'active') {
statusText.textContent = `Instance is healthy (Attempt ${currentAttempt}/${maxRetries})`; const elapsedTime = Math.round((Date.now() - startTime) / 1000);
statusText.textContent = `Instance is healthy (Attempt ${currentAttempt}/${maxRetries}, ${elapsedTime}s elapsed)`;
// Update progress bar to 100%
const progressBar = document.getElementById('healthProgress');
const progressText = document.getElementById('healthProgressText');
if (progressBar && progressText) {
progressBar.style.width = '100%';
progressBar.textContent = '100%';
progressBar.classList.remove('progress-bar-animated');
progressBar.classList.add('bg-success');
progressText.textContent = `Health check completed successfully in ${elapsedTime}s`;
}
return { return {
success: true, success: true,
data: data data: data,
attempts: currentAttempt,
elapsedTime: elapsedTime
}; };
} else { } else {
throw new Error('Instance is not healthy'); throw new Error('Instance is not healthy');
@@ -2297,25 +2306,58 @@ async function checkInstanceHealth(instanceUrl) {
} catch (error) { } catch (error) {
console.error(`Health check attempt ${currentAttempt} failed:`, error); console.error(`Health check attempt ${currentAttempt} failed:`, error);
// Update status to show current attempt // Update status to show current attempt and elapsed time
const healthStep = document.querySelectorAll('.step-item')[8]; const healthStepElement = document.querySelectorAll('.step-item')[10];
const statusText = healthStep.querySelector('.step-status'); const statusText = healthStepElement.querySelector('.step-status');
statusText.textContent = `Health check failed (Attempt ${currentAttempt}/${maxRetries}): ${error.message}`; const elapsedTime = Math.round((Date.now() - startTime) / 1000);
statusText.textContent = `Health check failed (Attempt ${currentAttempt}/${maxRetries}, ${elapsedTime}s elapsed): ${error.message}`;
if (currentAttempt === maxRetries) { // Update progress bar
const progressBar = document.getElementById('healthProgress');
const progressText = document.getElementById('healthProgressText');
if (progressBar && progressText) {
const progressPercent = Math.min((currentAttempt / maxRetries) * 100, 100);
progressBar.style.width = `${progressPercent}%`;
progressBar.textContent = `${Math.round(progressPercent)}%`;
progressText.textContent = `Attempt ${currentAttempt}/${maxRetries} (${elapsedTime}s elapsed)`;
}
if (currentAttempt === maxRetries || (Date.now() - startTime > maxTotalTime)) {
// Update progress bar to show failure
if (progressBar && progressText) {
progressBar.classList.remove('progress-bar-animated');
progressBar.classList.add('bg-danger');
progressText.textContent = `Health check failed after ${currentAttempt} attempts (${elapsedTime}s)`;
}
return { return {
success: false, success: false,
error: `Health check failed after ${maxRetries} attempts: ${error.message}` error: `Health check failed after ${currentAttempt} attempts (${elapsedTime}s): ${error.message}`
}; };
} }
// Wait 5 seconds before next attempt // Wait before next attempt (5 seconds base delay)
await new Promise(resolve => setTimeout(resolve, 5000)); await new Promise(resolve => setTimeout(resolve, baseDelay));
currentAttempt++; currentAttempt++;
// Update progress bar in real-time
updateHealthProgress(currentAttempt, maxRetries, elapsedTime);
} }
} }
} }
function updateHealthProgress(currentAttempt, maxRetries, elapsedTime) {
const progressBar = document.getElementById('healthProgress');
const progressText = document.getElementById('healthProgressText');
if (progressBar && progressText) {
const progressPercent = Math.min((currentAttempt / maxRetries) * 100, 100);
progressBar.style.width = `${progressPercent}%`;
progressBar.textContent = `${Math.round(progressPercent)}%`;
progressText.textContent = `Attempt ${currentAttempt}/${maxRetries} (${elapsedTime}s elapsed)`;
}
}
async function authenticateInstance(instanceUrl, instanceId) { async function authenticateInstance(instanceUrl, instanceId) {
try { try {
// First check if instance is already authenticated // First check if instance is already authenticated