better timeouts
This commit is contained in:
Binary file not shown.
@@ -350,7 +350,7 @@ def init_routes(main_bp):
|
|||||||
try:
|
try:
|
||||||
# Construct the health check URL
|
# Construct the health check URL
|
||||||
health_url = f"{instance.main_url.rstrip('/')}/health"
|
health_url = f"{instance.main_url.rstrip('/')}/health"
|
||||||
response = requests.get(health_url, timeout=5)
|
response = requests.get(health_url, timeout=30) # Increased timeout to 30 seconds
|
||||||
|
|
||||||
if response.status_code == 200:
|
if response.status_code == 200:
|
||||||
data = response.json()
|
data = response.json()
|
||||||
@@ -397,7 +397,7 @@ def init_routes(main_bp):
|
|||||||
deployed_commit = None
|
deployed_commit = None
|
||||||
try:
|
try:
|
||||||
version_url = f"{instance.main_url.rstrip('/')}/api/version"
|
version_url = f"{instance.main_url.rstrip('/')}/api/version"
|
||||||
resp = requests.get(version_url, timeout=5)
|
resp = requests.get(version_url, timeout=30) # Increased timeout to 30 seconds
|
||||||
if resp.status_code == 200:
|
if resp.status_code == 200:
|
||||||
version_data = resp.json()
|
version_data = resp.json()
|
||||||
deployed_version = version_data.get('version', 'unknown')
|
deployed_version = version_data.get('version', 'unknown')
|
||||||
@@ -419,7 +419,7 @@ def init_routes(main_bp):
|
|||||||
headers = {'Accept': 'application/json', 'Authorization': f'token {gitea_token}'}
|
headers = {'Accept': 'application/json', 'Authorization': f'token {gitea_token}'}
|
||||||
# Gitea API: /api/v1/repos/{owner}/{repo}/commits/{branch}
|
# Gitea API: /api/v1/repos/{owner}/{repo}/commits/{branch}
|
||||||
commit_url = f"{gitea_url}/api/v1/repos/{gitea_repo}/commits/{deployed_branch}"
|
commit_url = f"{gitea_url}/api/v1/repos/{gitea_repo}/commits/{deployed_branch}"
|
||||||
commit_resp = requests.get(commit_url, headers=headers, timeout=5)
|
commit_resp = requests.get(commit_url, headers=headers, timeout=30) # Increased timeout to 30 seconds
|
||||||
if commit_resp.status_code == 200:
|
if commit_resp.status_code == 200:
|
||||||
latest_version = commit_resp.json().get('sha')
|
latest_version = commit_resp.json().get('sha')
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
|
|||||||
@@ -571,40 +571,26 @@ async function startLaunch(data) {
|
|||||||
saveDataStep.classList.add('completed');
|
saveDataStep.classList.add('completed');
|
||||||
saveDataStep.querySelector('.step-status').textContent = 'Successfully saved instance data';
|
saveDataStep.querySelector('.step-status').textContent = 'Successfully saved instance data';
|
||||||
|
|
||||||
// Add instance details
|
|
||||||
const instanceDetails = document.createElement('div');
|
|
||||||
instanceDetails.className = 'mt-3';
|
|
||||||
instanceDetails.innerHTML = `
|
|
||||||
<div class="card">
|
|
||||||
<div class="card-body">
|
|
||||||
<h6 class="card-title mb-3">Instance Information</h6>
|
|
||||||
<div class="table-responsive">
|
|
||||||
<table class="table table-sm">
|
|
||||||
<thead>
|
|
||||||
<tr>
|
|
||||||
<th>Property</th>
|
|
||||||
<th>Value</th>
|
|
||||||
</tr>
|
|
||||||
</thead>
|
|
||||||
<tbody>
|
|
||||||
<tr>
|
|
||||||
<td>Internal Port</td>
|
|
||||||
<td>${data.port}</td>
|
|
||||||
</tr>
|
|
||||||
<tr>
|
|
||||||
<td>Domains</td>
|
|
||||||
<td><a href="https://${data.webAddresses.join(', ')}" target="_blank">${data.webAddresses.join(', ')}</a></td>
|
|
||||||
</tr>
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
`;
|
|
||||||
saveDataStep.querySelector('.step-content').appendChild(instanceDetails);
|
|
||||||
|
|
||||||
// After saving instance data, add the health check step
|
// After saving instance data, add the health check step
|
||||||
await updateStep(11, 'Health Check', 'Verifying instance health...');
|
await updateStep(11, 'Health Check', 'Verifying instance health...');
|
||||||
|
|
||||||
|
// Add a progress indicator
|
||||||
|
const healthStepElement = document.querySelectorAll('.step-item')[10];
|
||||||
|
const progressDiv = document.createElement('div');
|
||||||
|
progressDiv.className = 'mt-2';
|
||||||
|
progressDiv.innerHTML = `
|
||||||
|
<div class="progress" style="height: 20px;">
|
||||||
|
<div class="progress-bar progress-bar-striped progress-bar-animated"
|
||||||
|
role="progressbar"
|
||||||
|
style="width: 0%"
|
||||||
|
id="healthProgress">
|
||||||
|
0%
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<small class="text-muted" id="healthProgressText">Starting health check...</small>
|
||||||
|
`;
|
||||||
|
healthStepElement.querySelector('.step-content').appendChild(progressDiv);
|
||||||
|
|
||||||
const healthResult = await checkInstanceHealth(`https://${data.webAddresses[0]}`);
|
const healthResult = await checkInstanceHealth(`https://${data.webAddresses[0]}`);
|
||||||
|
|
||||||
if (!healthResult.success) {
|
if (!healthResult.success) {
|
||||||
@@ -612,7 +598,7 @@ async function startLaunch(data) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Add a retry button if health check fails
|
// Add a retry button if health check fails
|
||||||
const healthStep = document.querySelectorAll('.step-item')[10];
|
const healthStepElement2 = document.querySelectorAll('.step-item')[10];
|
||||||
if (!healthResult.success) {
|
if (!healthResult.success) {
|
||||||
const retryButton = document.createElement('button');
|
const retryButton = document.createElement('button');
|
||||||
retryButton.className = 'btn btn-sm btn-warning mt-2';
|
retryButton.className = 'btn btn-sm btn-warning mt-2';
|
||||||
@@ -622,15 +608,15 @@ async function startLaunch(data) {
|
|||||||
retryButton.innerHTML = '<i class="fas fa-spinner fa-spin me-1"></i> Checking...';
|
retryButton.innerHTML = '<i class="fas fa-spinner fa-spin me-1"></i> Checking...';
|
||||||
const retryResult = await checkInstanceHealth(`https://${data.webAddresses[0]}`);
|
const retryResult = await checkInstanceHealth(`https://${data.webAddresses[0]}`);
|
||||||
if (retryResult.success) {
|
if (retryResult.success) {
|
||||||
healthStep.classList.remove('failed');
|
healthStepElement2.classList.remove('failed');
|
||||||
healthStep.classList.add('completed');
|
healthStepElement2.classList.add('completed');
|
||||||
retryButton.remove();
|
retryButton.remove();
|
||||||
} else {
|
} else {
|
||||||
retryButton.disabled = false;
|
retryButton.disabled = false;
|
||||||
retryButton.innerHTML = '<i class="fas fa-sync-alt me-1"></i> Retry Health Check';
|
retryButton.innerHTML = '<i class="fas fa-sync-alt me-1"></i> Retry Health Check';
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
healthStep.querySelector('.step-content').appendChild(retryButton);
|
healthStepElement2.querySelector('.step-content').appendChild(retryButton);
|
||||||
}
|
}
|
||||||
|
|
||||||
// After health check, add authentication step
|
// After health check, add authentication step
|
||||||
@@ -2242,11 +2228,19 @@ async function saveInstanceData(instanceData) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function checkInstanceHealth(instanceUrl) {
|
async function checkInstanceHealth(instanceUrl) {
|
||||||
const maxRetries = 5;
|
const maxRetries = 120; // 120 retries * 5 seconds = 10 minutes total
|
||||||
|
const baseDelay = 5000; // 5 seconds base delay
|
||||||
let currentAttempt = 1;
|
let currentAttempt = 1;
|
||||||
|
const startTime = Date.now();
|
||||||
|
const maxTotalTime = 10 * 60 * 1000; // 10 minutes in milliseconds
|
||||||
|
|
||||||
while (currentAttempt <= maxRetries) {
|
while (currentAttempt <= maxRetries) {
|
||||||
try {
|
try {
|
||||||
|
// Check if we've exceeded the total timeout
|
||||||
|
if (Date.now() - startTime > maxTotalTime) {
|
||||||
|
throw new Error('Health check timeout: 10 minutes exceeded');
|
||||||
|
}
|
||||||
|
|
||||||
// First get the instance ID from the database
|
// First get the instance ID from the database
|
||||||
const response = await fetch('/instances');
|
const response = await fetch('/instances');
|
||||||
const text = await response.text();
|
const text = await response.text();
|
||||||
@@ -2280,16 +2274,31 @@ async function checkInstanceHealth(instanceUrl) {
|
|||||||
const data = await statusResponse.json();
|
const data = await statusResponse.json();
|
||||||
|
|
||||||
// Update the health check step
|
// Update the health check step
|
||||||
const healthStep = document.querySelectorAll('.step-item')[8]; // Adjust index based on your steps
|
const healthStepElement = document.querySelectorAll('.step-item')[10]; // Adjust index based on your steps
|
||||||
healthStep.classList.remove('active');
|
healthStepElement.classList.remove('active');
|
||||||
healthStep.classList.add('completed');
|
healthStepElement.classList.add('completed');
|
||||||
const statusText = healthStep.querySelector('.step-status');
|
const statusText = healthStepElement.querySelector('.step-status');
|
||||||
|
|
||||||
if (data.status === 'active') {
|
if (data.status === 'active') {
|
||||||
statusText.textContent = `Instance is healthy (Attempt ${currentAttempt}/${maxRetries})`;
|
const elapsedTime = Math.round((Date.now() - startTime) / 1000);
|
||||||
|
statusText.textContent = `Instance is healthy (Attempt ${currentAttempt}/${maxRetries}, ${elapsedTime}s elapsed)`;
|
||||||
|
|
||||||
|
// Update progress bar to 100%
|
||||||
|
const progressBar = document.getElementById('healthProgress');
|
||||||
|
const progressText = document.getElementById('healthProgressText');
|
||||||
|
if (progressBar && progressText) {
|
||||||
|
progressBar.style.width = '100%';
|
||||||
|
progressBar.textContent = '100%';
|
||||||
|
progressBar.classList.remove('progress-bar-animated');
|
||||||
|
progressBar.classList.add('bg-success');
|
||||||
|
progressText.textContent = `Health check completed successfully in ${elapsedTime}s`;
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
success: true,
|
success: true,
|
||||||
data: data
|
data: data,
|
||||||
|
attempts: currentAttempt,
|
||||||
|
elapsedTime: elapsedTime
|
||||||
};
|
};
|
||||||
} else {
|
} else {
|
||||||
throw new Error('Instance is not healthy');
|
throw new Error('Instance is not healthy');
|
||||||
@@ -2297,25 +2306,58 @@ async function checkInstanceHealth(instanceUrl) {
|
|||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.error(`Health check attempt ${currentAttempt} failed:`, error);
|
console.error(`Health check attempt ${currentAttempt} failed:`, error);
|
||||||
|
|
||||||
// Update status to show current attempt
|
// Update status to show current attempt and elapsed time
|
||||||
const healthStep = document.querySelectorAll('.step-item')[8];
|
const healthStepElement = document.querySelectorAll('.step-item')[10];
|
||||||
const statusText = healthStep.querySelector('.step-status');
|
const statusText = healthStepElement.querySelector('.step-status');
|
||||||
statusText.textContent = `Health check failed (Attempt ${currentAttempt}/${maxRetries}): ${error.message}`;
|
const elapsedTime = Math.round((Date.now() - startTime) / 1000);
|
||||||
|
statusText.textContent = `Health check failed (Attempt ${currentAttempt}/${maxRetries}, ${elapsedTime}s elapsed): ${error.message}`;
|
||||||
|
|
||||||
if (currentAttempt === maxRetries) {
|
// Update progress bar
|
||||||
|
const progressBar = document.getElementById('healthProgress');
|
||||||
|
const progressText = document.getElementById('healthProgressText');
|
||||||
|
if (progressBar && progressText) {
|
||||||
|
const progressPercent = Math.min((currentAttempt / maxRetries) * 100, 100);
|
||||||
|
progressBar.style.width = `${progressPercent}%`;
|
||||||
|
progressBar.textContent = `${Math.round(progressPercent)}%`;
|
||||||
|
progressText.textContent = `Attempt ${currentAttempt}/${maxRetries} (${elapsedTime}s elapsed)`;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (currentAttempt === maxRetries || (Date.now() - startTime > maxTotalTime)) {
|
||||||
|
// Update progress bar to show failure
|
||||||
|
if (progressBar && progressText) {
|
||||||
|
progressBar.classList.remove('progress-bar-animated');
|
||||||
|
progressBar.classList.add('bg-danger');
|
||||||
|
progressText.textContent = `Health check failed after ${currentAttempt} attempts (${elapsedTime}s)`;
|
||||||
|
}
|
||||||
|
|
||||||
return {
|
return {
|
||||||
success: false,
|
success: false,
|
||||||
error: `Health check failed after ${maxRetries} attempts: ${error.message}`
|
error: `Health check failed after ${currentAttempt} attempts (${elapsedTime}s): ${error.message}`
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
// Wait 5 seconds before next attempt
|
// Wait before next attempt (5 seconds base delay)
|
||||||
await new Promise(resolve => setTimeout(resolve, 5000));
|
await new Promise(resolve => setTimeout(resolve, baseDelay));
|
||||||
currentAttempt++;
|
currentAttempt++;
|
||||||
|
|
||||||
|
// Update progress bar in real-time
|
||||||
|
updateHealthProgress(currentAttempt, maxRetries, elapsedTime);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function updateHealthProgress(currentAttempt, maxRetries, elapsedTime) {
|
||||||
|
const progressBar = document.getElementById('healthProgress');
|
||||||
|
const progressText = document.getElementById('healthProgressText');
|
||||||
|
|
||||||
|
if (progressBar && progressText) {
|
||||||
|
const progressPercent = Math.min((currentAttempt / maxRetries) * 100, 100);
|
||||||
|
progressBar.style.width = `${progressPercent}%`;
|
||||||
|
progressBar.textContent = `${Math.round(progressPercent)}%`;
|
||||||
|
progressText.textContent = `Attempt ${currentAttempt}/${maxRetries} (${elapsedTime}s elapsed)`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
async function authenticateInstance(instanceUrl, instanceId) {
|
async function authenticateInstance(instanceUrl, instanceId) {
|
||||||
try {
|
try {
|
||||||
// First check if instance is already authenticated
|
// First check if instance is already authenticated
|
||||||
|
|||||||
Reference in New Issue
Block a user