better 504 handling
This commit is contained in:
@@ -849,9 +849,12 @@ def deploy_stack():
|
|||||||
if stack['Name'] == data['name']:
|
if stack['Name'] == data['name']:
|
||||||
current_app.logger.info(f"Found existing stack: {stack['Name']} (ID: {stack['Id']})")
|
current_app.logger.info(f"Found existing stack: {stack['Name']} (ID: {stack['Id']})")
|
||||||
return jsonify({
|
return jsonify({
|
||||||
'name': stack['Name'],
|
'success': True,
|
||||||
'id': stack['Id'],
|
'data': {
|
||||||
'status': 'existing'
|
'name': stack['Name'],
|
||||||
|
'id': stack['Id'],
|
||||||
|
'status': 'existing'
|
||||||
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
# If no existing stack found, proceed with creation
|
# If no existing stack found, proceed with creation
|
||||||
@@ -864,7 +867,7 @@ def deploy_stack():
|
|||||||
# Add endpointId as a query parameter
|
# Add endpointId as a query parameter
|
||||||
params = {'endpointId': endpoint_id}
|
params = {'endpointId': endpoint_id}
|
||||||
|
|
||||||
# Set a longer timeout for stack creation (10 minutes)
|
# Use a shorter timeout for stack creation initiation (2 minutes)
|
||||||
create_response = requests.post(
|
create_response = requests.post(
|
||||||
url,
|
url,
|
||||||
headers={
|
headers={
|
||||||
@@ -874,7 +877,7 @@ def deploy_stack():
|
|||||||
},
|
},
|
||||||
params=params,
|
params=params,
|
||||||
json=request_body,
|
json=request_body,
|
||||||
timeout=600 # 10 minutes timeout for stack creation
|
timeout=120 # 2 minutes timeout for stack creation initiation
|
||||||
)
|
)
|
||||||
|
|
||||||
# Log the response details
|
# Log the response details
|
||||||
@@ -894,15 +897,20 @@ def deploy_stack():
|
|||||||
return jsonify({'error': f'Failed to create stack: {error_message}'}), 500
|
return jsonify({'error': f'Failed to create stack: {error_message}'}), 500
|
||||||
|
|
||||||
stack_info = create_response.json()
|
stack_info = create_response.json()
|
||||||
|
current_app.logger.info(f"Stack creation initiated: {stack_info['Name']} (ID: {stack_info['Id']})")
|
||||||
|
|
||||||
return jsonify({
|
return jsonify({
|
||||||
'name': stack_info['Name'],
|
'success': True,
|
||||||
'id': stack_info['Id'],
|
'data': {
|
||||||
'status': 'created'
|
'name': stack_info['Name'],
|
||||||
|
'id': stack_info['Id'],
|
||||||
|
'status': 'creating'
|
||||||
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
except requests.exceptions.Timeout:
|
except requests.exceptions.Timeout:
|
||||||
current_app.logger.error("Request timed out while deploying stack")
|
current_app.logger.error("Request timed out while initiating stack deployment")
|
||||||
return jsonify({'error': 'Request timed out while deploying stack. The operation may still be in progress.'}), 504
|
return jsonify({'error': 'Request timed out while initiating stack deployment. The operation may still be in progress.'}), 504
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
current_app.logger.error(f"Error deploying stack: {str(e)}")
|
current_app.logger.error(f"Error deploying stack: {str(e)}")
|
||||||
return jsonify({'error': str(e)}), 500
|
return jsonify({'error': str(e)}), 500
|
||||||
|
|||||||
@@ -481,6 +481,24 @@ async function startLaunch(data) {
|
|||||||
|
|
||||||
// Step 9: Deploy Stack
|
// Step 9: Deploy Stack
|
||||||
await updateStep(9, 'Deploying Stack', 'Launching your application stack...');
|
await updateStep(9, 'Deploying Stack', 'Launching your application stack...');
|
||||||
|
|
||||||
|
// Add progress indicator for stack deployment
|
||||||
|
const stackDeployStepElement = document.querySelectorAll('.step-item')[8];
|
||||||
|
const stackProgressDiv = document.createElement('div');
|
||||||
|
stackProgressDiv.className = 'mt-2';
|
||||||
|
stackProgressDiv.innerHTML = `
|
||||||
|
<div class="progress" style="height: 20px;">
|
||||||
|
<div class="progress-bar progress-bar-striped progress-bar-animated"
|
||||||
|
role="progressbar"
|
||||||
|
style="width: 0%"
|
||||||
|
id="stackProgress">
|
||||||
|
0%
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<small class="text-muted" id="stackProgressText">Initiating stack deployment...</small>
|
||||||
|
`;
|
||||||
|
stackDeployStepElement.querySelector('.step-content').appendChild(stackProgressDiv);
|
||||||
|
|
||||||
const stackResult = await deployStack(dockerComposeResult.content, data.instanceName, data.port);
|
const stackResult = await deployStack(dockerComposeResult.content, data.instanceName, data.port);
|
||||||
launchReport.steps.push({
|
launchReport.steps.push({
|
||||||
step: 'Stack Deployment',
|
step: 'Stack Deployment',
|
||||||
@@ -498,6 +516,8 @@ async function startLaunch(data) {
|
|||||||
stackDeployStep.querySelector('.step-status').textContent =
|
stackDeployStep.querySelector('.step-status').textContent =
|
||||||
stackResult.data.status === 'existing' ?
|
stackResult.data.status === 'existing' ?
|
||||||
'Using existing stack' :
|
'Using existing stack' :
|
||||||
|
stackResult.data.status === 'active' ?
|
||||||
|
'Successfully deployed and activated stack' :
|
||||||
'Successfully deployed stack';
|
'Successfully deployed stack';
|
||||||
|
|
||||||
// Add stack details
|
// Add stack details
|
||||||
@@ -522,13 +542,13 @@ async function startLaunch(data) {
|
|||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td>Stack ID</td>
|
<td>Stack ID</td>
|
||||||
<td>${stackResult.data.id}</td>
|
<td>${stackResult.data.id || 'Will be determined during deployment'}</td>
|
||||||
</tr>
|
</tr>
|
||||||
<tr>
|
<tr>
|
||||||
<td>Status</td>
|
<td>Status</td>
|
||||||
<td>
|
<td>
|
||||||
<span class="badge bg-${stackResult.data.status === 'existing' ? 'info' : 'success'}">
|
<span class="badge bg-${stackResult.data.status === 'existing' ? 'info' : stackResult.data.status === 'active' ? 'success' : 'warning'}">
|
||||||
${stackResult.data.status === 'existing' ? 'Existing' : 'Deployed'}
|
${stackResult.data.status === 'existing' ? 'Existing' : stackResult.data.status === 'active' ? 'Active' : 'Deployed'}
|
||||||
</span>
|
</span>
|
||||||
</td>
|
</td>
|
||||||
</tr>
|
</tr>
|
||||||
@@ -547,7 +567,7 @@ async function startLaunch(data) {
|
|||||||
name: data.instanceName,
|
name: data.instanceName,
|
||||||
port: data.port,
|
port: data.port,
|
||||||
domains: data.webAddresses,
|
domains: data.webAddresses,
|
||||||
stack_id: stackResult.data.id,
|
stack_id: stackResult.data.id || null, // May be null if we got a 504
|
||||||
stack_name: stackResult.data.name,
|
stack_name: stackResult.data.name,
|
||||||
status: stackResult.data.status,
|
status: stackResult.data.status,
|
||||||
repository: data.repository,
|
repository: data.repository,
|
||||||
@@ -2006,7 +2026,7 @@ async function saveInstanceData(instanceData) {
|
|||||||
main_url: `https://${instanceData.domains[0]}`,
|
main_url: `https://${instanceData.domains[0]}`,
|
||||||
status: 'inactive',
|
status: 'inactive',
|
||||||
port: instanceData.port,
|
port: instanceData.port,
|
||||||
stack_id: instanceData.stack_id,
|
stack_id: instanceData.stack_id || '', // Use empty string if null
|
||||||
stack_name: instanceData.stack_name,
|
stack_name: instanceData.stack_name,
|
||||||
repository: instanceData.repository,
|
repository: instanceData.repository,
|
||||||
branch: instanceData.branch
|
branch: instanceData.branch
|
||||||
@@ -2031,7 +2051,7 @@ async function saveInstanceData(instanceData) {
|
|||||||
main_url: `https://${instanceData.domains[0]}`,
|
main_url: `https://${instanceData.domains[0]}`,
|
||||||
status: 'inactive',
|
status: 'inactive',
|
||||||
port: instanceData.port,
|
port: instanceData.port,
|
||||||
stack_id: instanceData.stack_id,
|
stack_id: instanceData.stack_id || '', // Use empty string if null
|
||||||
stack_name: instanceData.stack_name,
|
stack_name: instanceData.stack_name,
|
||||||
repository: instanceData.repository,
|
repository: instanceData.repository,
|
||||||
branch: instanceData.branch
|
branch: instanceData.branch
|
||||||
@@ -2536,9 +2556,6 @@ async function checkStackExists(stackName) {
|
|||||||
async function deployStack(dockerComposeContent, stackName, port) {
|
async function deployStack(dockerComposeContent, stackName, port) {
|
||||||
try {
|
try {
|
||||||
// First, attempt to deploy the stack
|
// First, attempt to deploy the stack
|
||||||
const controller = new AbortController();
|
|
||||||
const timeoutId = setTimeout(() => controller.abort(), 10 * 60 * 1000); // 10 minutes timeout
|
|
||||||
|
|
||||||
const response = await fetch('/api/admin/deploy-stack', {
|
const response = await fetch('/api/admin/deploy-stack', {
|
||||||
method: 'POST',
|
method: 'POST',
|
||||||
headers: {
|
headers: {
|
||||||
@@ -2574,22 +2591,41 @@ async function deployStack(dockerComposeContent, stackName, port) {
|
|||||||
value: new Date().toISOString()
|
value: new Date().toISOString()
|
||||||
}
|
}
|
||||||
]
|
]
|
||||||
}),
|
})
|
||||||
signal: controller.signal
|
|
||||||
});
|
});
|
||||||
|
|
||||||
clearTimeout(timeoutId); // Clear the timeout if the request completes
|
// Handle 504 Gateway Timeout as successful initiation
|
||||||
|
if (response.status === 504) {
|
||||||
|
console.log('Received 504 Gateway Timeout - stack creation may still be in progress');
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
name: stackName,
|
||||||
|
id: null, // Will be determined during polling
|
||||||
|
status: 'creating'
|
||||||
|
}
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!response.ok) {
|
||||||
const error = await response.json();
|
const error = await response.json();
|
||||||
throw new Error(error.error || 'Failed to deploy stack');
|
throw new Error(error.error || 'Failed to deploy stack');
|
||||||
}
|
}
|
||||||
|
|
||||||
// Return success result with response data
|
|
||||||
const result = await response.json();
|
const result = await response.json();
|
||||||
|
console.log('Stack deployment initiated:', result);
|
||||||
|
|
||||||
|
// If stack is being created, poll for status
|
||||||
|
if (result.data.status === 'creating') {
|
||||||
|
console.log('Stack is being created, polling for status...');
|
||||||
|
const pollResult = await pollStackStatus(stackName, 10 * 60 * 1000); // 10 minutes max
|
||||||
|
return pollResult;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return success result with response data
|
||||||
return {
|
return {
|
||||||
success: true,
|
success: true,
|
||||||
data: result
|
data: result.data
|
||||||
};
|
};
|
||||||
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -2600,3 +2636,112 @@ async function deployStack(dockerComposeContent, stackName, port) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Function to poll stack status
|
||||||
|
async function pollStackStatus(stackName, maxWaitTime = 10 * 60 * 1000) {
|
||||||
|
const startTime = Date.now();
|
||||||
|
const pollInterval = 5000; // 5 seconds
|
||||||
|
let attempts = 0;
|
||||||
|
|
||||||
|
console.log(`Starting to poll stack status for: ${stackName}`);
|
||||||
|
|
||||||
|
// Update progress indicator
|
||||||
|
const progressBar = document.getElementById('stackProgress');
|
||||||
|
const progressText = document.getElementById('stackProgressText');
|
||||||
|
|
||||||
|
while (Date.now() - startTime < maxWaitTime) {
|
||||||
|
attempts++;
|
||||||
|
console.log(`Polling attempt ${attempts} for stack: ${stackName}`);
|
||||||
|
|
||||||
|
// Update progress
|
||||||
|
const elapsed = Date.now() - startTime;
|
||||||
|
const progress = Math.min((elapsed / maxWaitTime) * 100, 95); // Cap at 95% until complete
|
||||||
|
if (progressBar && progressText) {
|
||||||
|
progressBar.style.width = `${progress}%`;
|
||||||
|
progressBar.textContent = `${Math.round(progress)}%`;
|
||||||
|
progressText.textContent = `Checking stack status (attempt ${attempts})...`;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const response = await fetch('/api/admin/check-stack-status', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: {
|
||||||
|
'Content-Type': 'application/json',
|
||||||
|
'X-CSRF-Token': document.querySelector('meta[name="csrf-token"]').content
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
stack_name: stackName
|
||||||
|
})
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.ok) {
|
||||||
|
const result = await response.json();
|
||||||
|
console.log(`Stack status check result:`, result);
|
||||||
|
|
||||||
|
if (result.data && result.data.status === 'active') {
|
||||||
|
console.log(`Stack ${stackName} is now active!`);
|
||||||
|
// Update progress to 100%
|
||||||
|
if (progressBar && progressText) {
|
||||||
|
progressBar.style.width = '100%';
|
||||||
|
progressBar.textContent = '100%';
|
||||||
|
progressText.textContent = 'Stack is now active!';
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
success: true,
|
||||||
|
data: {
|
||||||
|
name: stackName,
|
||||||
|
id: result.data.stack_id,
|
||||||
|
status: 'active'
|
||||||
|
}
|
||||||
|
};
|
||||||
|
} else if (result.data && result.data.status === 'partial') {
|
||||||
|
console.log(`Stack ${stackName} is partially running, continuing to poll...`);
|
||||||
|
if (progressText) {
|
||||||
|
progressText.textContent = `Stack is partially running (attempt ${attempts})...`;
|
||||||
|
}
|
||||||
|
} else if (result.data && result.data.status === 'inactive') {
|
||||||
|
console.log(`Stack ${stackName} is inactive, continuing to poll...`);
|
||||||
|
if (progressText) {
|
||||||
|
progressText.textContent = `Stack is starting up (attempt ${attempts})...`;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.log(`Stack ${stackName} status unknown, continuing to poll...`);
|
||||||
|
if (progressText) {
|
||||||
|
progressText.textContent = `Checking stack status (attempt ${attempts})...`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else if (response.status === 404) {
|
||||||
|
console.log(`Stack ${stackName} not found yet, continuing to poll...`);
|
||||||
|
if (progressText) {
|
||||||
|
progressText.textContent = `Stack not found yet (attempt ${attempts})...`;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
console.log(`Stack status check failed with status ${response.status}, continuing to poll...`);
|
||||||
|
if (progressText) {
|
||||||
|
progressText.textContent = `Status check failed (attempt ${attempts})...`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
console.error(`Error polling stack status (attempt ${attempts}):`, error);
|
||||||
|
if (progressText) {
|
||||||
|
progressText.textContent = `Error checking status (attempt ${attempts})...`;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wait before next poll
|
||||||
|
await new Promise(resolve => setTimeout(resolve, pollInterval));
|
||||||
|
}
|
||||||
|
|
||||||
|
// If we get here, we've timed out
|
||||||
|
console.error(`Stack status polling timed out after ${maxWaitTime / 1000} seconds`);
|
||||||
|
if (progressBar && progressText) {
|
||||||
|
progressBar.style.width = '100%';
|
||||||
|
progressBar.classList.remove('progress-bar-animated');
|
||||||
|
progressBar.classList.add('bg-warning');
|
||||||
|
progressText.textContent = 'Stack deployment timed out';
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
success: false,
|
||||||
|
error: `Stack deployment timed out after ${maxWaitTime / 1000} seconds. The stack may still be deploying.`
|
||||||
|
};
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user