Data converter
This commit is contained in:
@@ -413,6 +413,20 @@ function setupNavigation() {
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// If switching to convert-data section, ensure event listeners are set up
|
||||
if (targetId === 'convert-data') {
|
||||
console.log('Convert Data section activated');
|
||||
// Initialize data convert functionality if available
|
||||
setTimeout(() => {
|
||||
if (typeof window.setupDataConvertEventListeners === 'function') {
|
||||
console.log('Setting up data convert event listeners...');
|
||||
window.setupDataConvertEventListeners();
|
||||
} else {
|
||||
console.warn('setupDataConvertEventListeners function not available');
|
||||
}
|
||||
}, 100); // Small delay to ensure DOM is ready
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
585
map/app/public/js/data-convert.js
Normal file
585
map/app/public/js/data-convert.js
Normal file
@@ -0,0 +1,585 @@
|
||||
let processingData = [];
|
||||
let resultsMap = null;
|
||||
let markers = [];
|
||||
|
||||
// Utility function to show status messages
|
||||
function showDataConvertStatus(message, type = 'info') {
|
||||
// Try to use the global showStatus from admin.js if available
|
||||
if (typeof window.showStatus === 'function') {
|
||||
return window.showStatus(message, type);
|
||||
}
|
||||
|
||||
// Fallback to console
|
||||
console.log(`[${type.toUpperCase()}] ${message}`);
|
||||
|
||||
// Try to display in status container if it exists
|
||||
const statusContainer = document.getElementById('status-container');
|
||||
if (statusContainer) {
|
||||
const statusEl = document.createElement('div');
|
||||
statusEl.className = `status-message status-${type}`;
|
||||
statusEl.textContent = message;
|
||||
statusContainer.appendChild(statusEl);
|
||||
|
||||
// Auto-remove after 5 seconds
|
||||
setTimeout(() => {
|
||||
if (statusEl.parentNode) {
|
||||
statusEl.parentNode.removeChild(statusEl);
|
||||
}
|
||||
}, 5000);
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize when DOM is loaded
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
console.log('Data convert JS loaded');
|
||||
// Don't auto-initialize, wait for section to be activated
|
||||
|
||||
// Make setupDataConvertEventListeners available globally for admin.js
|
||||
window.setupDataConvertEventListeners = setupDataConvertEventListeners;
|
||||
});
|
||||
|
||||
function setupDataConvertEventListeners() {
|
||||
console.log('Setting up data convert event listeners...');
|
||||
|
||||
const fileInput = document.getElementById('csv-file-input');
|
||||
const browseBtn = document.getElementById('browse-btn');
|
||||
const uploadArea = document.getElementById('upload-area');
|
||||
const uploadForm = document.getElementById('csv-upload-form');
|
||||
const clearBtn = document.getElementById('clear-upload-btn');
|
||||
const saveResultsBtn = document.getElementById('save-results-btn');
|
||||
const newUploadBtn = document.getElementById('new-upload-btn');
|
||||
|
||||
console.log('Elements found:', {
|
||||
fileInput: !!fileInput,
|
||||
browseBtn: !!browseBtn,
|
||||
uploadArea: !!uploadArea,
|
||||
uploadForm: !!uploadForm,
|
||||
clearBtn: !!clearBtn,
|
||||
saveResultsBtn: !!saveResultsBtn,
|
||||
newUploadBtn: !!newUploadBtn
|
||||
});
|
||||
|
||||
// File input change
|
||||
if (fileInput) {
|
||||
fileInput.addEventListener('change', handleFileSelect);
|
||||
}
|
||||
|
||||
// Browse button
|
||||
if (browseBtn) {
|
||||
browseBtn.addEventListener('click', () => {
|
||||
console.log('Browse button clicked');
|
||||
fileInput?.click();
|
||||
});
|
||||
}
|
||||
|
||||
// Drag and drop
|
||||
if (uploadArea) {
|
||||
uploadArea.addEventListener('dragover', handleDragOver);
|
||||
uploadArea.addEventListener('dragleave', handleDragLeave);
|
||||
uploadArea.addEventListener('drop', handleDrop);
|
||||
uploadArea.addEventListener('click', () => fileInput?.click());
|
||||
}
|
||||
|
||||
// Form submission
|
||||
if (uploadForm) {
|
||||
uploadForm.addEventListener('submit', handleCSVUpload);
|
||||
}
|
||||
|
||||
// Clear button
|
||||
if (clearBtn) {
|
||||
clearBtn.addEventListener('click', clearUpload);
|
||||
}
|
||||
|
||||
// Save results button - ADD DATA TO MAP
|
||||
if (saveResultsBtn) {
|
||||
saveResultsBtn.addEventListener('click', saveGeocodedResults);
|
||||
console.log('Save results button event listener attached');
|
||||
}
|
||||
|
||||
// New upload button
|
||||
if (newUploadBtn) {
|
||||
newUploadBtn.addEventListener('click', resetToUpload);
|
||||
}
|
||||
}
|
||||
|
||||
function handleFileSelect(e) {
|
||||
const file = e.target.files[0];
|
||||
if (file) {
|
||||
displayFileInfo(file);
|
||||
}
|
||||
}
|
||||
|
||||
function handleDragOver(e) {
|
||||
e.preventDefault();
|
||||
e.currentTarget.classList.add('drag-over');
|
||||
}
|
||||
|
||||
function handleDragLeave(e) {
|
||||
e.preventDefault();
|
||||
e.currentTarget.classList.remove('drag-over');
|
||||
}
|
||||
|
||||
function handleDrop(e) {
|
||||
e.preventDefault();
|
||||
e.currentTarget.classList.remove('drag-over');
|
||||
|
||||
const files = e.dataTransfer.files;
|
||||
if (files.length > 0) {
|
||||
const file = files[0];
|
||||
if (file.type === 'text/csv' || file.name.endsWith('.csv')) {
|
||||
document.getElementById('csv-file-input').files = files;
|
||||
displayFileInfo(file);
|
||||
} else {
|
||||
showDataConvertStatus('Please upload a CSV file', 'error');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function displayFileInfo(file) {
|
||||
document.getElementById('file-info').style.display = 'block';
|
||||
document.getElementById('file-name').textContent = file.name;
|
||||
document.getElementById('file-size').textContent = formatFileSize(file.size);
|
||||
document.getElementById('process-csv-btn').disabled = false;
|
||||
}
|
||||
|
||||
function formatFileSize(bytes) {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
const k = 1024;
|
||||
const sizes = ['Bytes', 'KB', 'MB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
||||
}
|
||||
|
||||
async function handleCSVUpload(e) {
|
||||
e.preventDefault();
|
||||
|
||||
const fileInput = document.getElementById('csv-file-input');
|
||||
const file = fileInput?.files[0];
|
||||
|
||||
if (!file) {
|
||||
showDataConvertStatus('Please select a CSV file', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Reset processing data
|
||||
processingData = [];
|
||||
|
||||
// Show processing section
|
||||
const uploadSection = document.getElementById('upload-section');
|
||||
const processingSection = document.getElementById('processing-section');
|
||||
|
||||
if (!uploadSection || !processingSection) {
|
||||
console.error('Required DOM elements not found');
|
||||
showDataConvertStatus('Interface error: required elements not found', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
uploadSection.style.display = 'none';
|
||||
processingSection.style.display = 'block';
|
||||
|
||||
// Initialize results map
|
||||
initializeResultsMap();
|
||||
|
||||
// Create form data
|
||||
const formData = new FormData();
|
||||
formData.append('csvFile', file);
|
||||
|
||||
try {
|
||||
// Send the file and handle SSE response
|
||||
const response = await fetch('/api/admin/data-convert/process-csv', {
|
||||
method: 'POST',
|
||||
body: formData
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
// Read the response as a stream for SSE
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = ''; // Buffer to accumulate partial data
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
// Decode the chunk and add to buffer
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
|
||||
// Split buffer by lines and process complete lines
|
||||
const lines = buffer.split('\n');
|
||||
|
||||
// Keep the last potentially incomplete line in the buffer
|
||||
buffer = lines.pop() || '';
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('data: ')) {
|
||||
try {
|
||||
const jsonData = line.substring(6).trim(); // Remove 'data: ' prefix and trim
|
||||
if (jsonData && jsonData !== '') {
|
||||
const data = JSON.parse(jsonData);
|
||||
handleProcessingUpdate(data);
|
||||
}
|
||||
} catch (parseError) {
|
||||
console.warn('Failed to parse SSE data:', parseError);
|
||||
console.warn('Problematic line:', line);
|
||||
console.warn('JSON data:', line.substring(6));
|
||||
}
|
||||
} else if (line.trim() === '') {
|
||||
// Empty line, ignore
|
||||
continue;
|
||||
} else if (line.trim() !== '' && !line.startsWith('event:') && !line.startsWith('id:')) {
|
||||
// Unexpected line format
|
||||
console.warn('Unexpected SSE line format:', line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process any remaining data in buffer
|
||||
if (buffer.trim()) {
|
||||
const line = buffer;
|
||||
if (line.startsWith('data: ')) {
|
||||
try {
|
||||
const jsonData = line.substring(6).trim();
|
||||
if (jsonData && jsonData !== '') {
|
||||
const data = JSON.parse(jsonData);
|
||||
handleProcessingUpdate(data);
|
||||
}
|
||||
} catch (parseError) {
|
||||
console.warn('Failed to parse final SSE data:', parseError);
|
||||
console.warn('Final buffer content:', buffer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('CSV processing error:', error);
|
||||
showDataConvertStatus(error.message || 'Failed to process CSV', 'error');
|
||||
resetToUpload();
|
||||
}
|
||||
}
|
||||
|
||||
// Enhanced processing update handler
|
||||
function handleProcessingUpdate(data) {
|
||||
console.log('Processing update:', data);
|
||||
|
||||
// Validate data structure
|
||||
if (!data || typeof data !== 'object') {
|
||||
console.warn('Invalid data received:', data);
|
||||
return;
|
||||
}
|
||||
|
||||
switch (data.type) {
|
||||
case 'start':
|
||||
updateProgress(0, data.total);
|
||||
document.getElementById('current-address').textContent = 'Starting geocoding process...';
|
||||
break;
|
||||
|
||||
case 'progress':
|
||||
updateProgress(data.current, data.total);
|
||||
document.getElementById('current-address').textContent = `Processing: ${data.address}`;
|
||||
break;
|
||||
|
||||
case 'geocoded':
|
||||
// Mark as successful and add to processing data
|
||||
const successData = { ...data.data, geocode_success: true };
|
||||
processingData.push(successData);
|
||||
addResultToTable(successData, 'success');
|
||||
addMarkerToMap(successData);
|
||||
updateProgress(data.index + 1, data.total);
|
||||
break;
|
||||
|
||||
case 'error':
|
||||
// Mark as failed and add to processing data
|
||||
const errorData = { ...data.data, geocode_success: false };
|
||||
processingData.push(errorData);
|
||||
addResultToTable(errorData, 'error');
|
||||
break;
|
||||
|
||||
case 'complete':
|
||||
console.log('Received complete event:', data);
|
||||
onProcessingComplete(data);
|
||||
break;
|
||||
|
||||
case 'fatal_error':
|
||||
showDataConvertStatus(data.message, 'error');
|
||||
resetToUpload();
|
||||
break;
|
||||
|
||||
default:
|
||||
console.warn('Unknown data type received:', data.type);
|
||||
}
|
||||
}
|
||||
|
||||
function updateProgress(current, total) {
|
||||
const percentage = (current / total) * 100;
|
||||
document.getElementById('progress-bar-fill').style.width = percentage + '%';
|
||||
document.getElementById('progress-current').textContent = current;
|
||||
document.getElementById('progress-total').textContent = total;
|
||||
}
|
||||
|
||||
function initializeResultsMap() {
|
||||
const mapContainer = document.getElementById('results-map');
|
||||
if (!mapContainer) return;
|
||||
|
||||
// Initialize map
|
||||
resultsMap = L.map('results-map').setView([53.5461, -113.4938], 11);
|
||||
|
||||
L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
|
||||
attribution: '© OpenStreetMap contributors'
|
||||
}).addTo(resultsMap);
|
||||
|
||||
document.getElementById('results-preview').style.display = 'block';
|
||||
|
||||
// Fix map sizing
|
||||
setTimeout(() => {
|
||||
resultsMap.invalidateSize();
|
||||
}, 100);
|
||||
}
|
||||
|
||||
function addResultToTable(data, status) {
|
||||
const tbody = document.getElementById('results-tbody');
|
||||
const row = document.createElement('tr');
|
||||
row.className = status === 'success' ? 'result-success' : 'result-error';
|
||||
|
||||
if (status === 'success') {
|
||||
row.innerHTML = `
|
||||
<td><span class="status-icon success">✓</span></td>
|
||||
<td>${escapeHtml(data.address || data.Address || '')}</td>
|
||||
<td>${escapeHtml(data.geocoded_address || '')}</td>
|
||||
<td>${data.latitude.toFixed(6)}, ${data.longitude.toFixed(6)}</td>
|
||||
`;
|
||||
} else {
|
||||
row.innerHTML = `
|
||||
<td><span class="status-icon error">✗</span></td>
|
||||
<td>${escapeHtml(data.address || '')}</td>
|
||||
<td colspan="2" class="error-message">${escapeHtml(data.error || 'Geocoding failed')}</td>
|
||||
`;
|
||||
}
|
||||
|
||||
tbody.appendChild(row);
|
||||
}
|
||||
|
||||
function addMarkerToMap(data) {
|
||||
if (!resultsMap || !data.latitude || !data.longitude) return;
|
||||
|
||||
const marker = L.marker([data.latitude, data.longitude])
|
||||
.bindPopup(`
|
||||
<strong>${escapeHtml(data.geocoded_address || data.address)}</strong><br>
|
||||
${data.latitude.toFixed(6)}, ${data.longitude.toFixed(6)}
|
||||
`);
|
||||
|
||||
marker.addTo(resultsMap);
|
||||
markers.push(marker);
|
||||
|
||||
// Adjust map bounds to show all markers
|
||||
if (markers.length > 0) {
|
||||
const group = new L.featureGroup(markers);
|
||||
resultsMap.fitBounds(group.getBounds().pad(0.1));
|
||||
}
|
||||
}
|
||||
|
||||
function onProcessingComplete(data) {
|
||||
console.log('Processing complete called with data:', data);
|
||||
|
||||
document.getElementById('current-address').textContent =
|
||||
`Complete! Processed ${data.processed || data.success || 0} addresses successfully, ${data.errors || data.failed || 0} errors.`;
|
||||
|
||||
const processedCount = data.processed || data.success || processingData.filter(item => item.geocode_success !== false).length;
|
||||
|
||||
if (processedCount > 0) {
|
||||
console.log('Showing processing actions for', processedCount, 'successful items');
|
||||
const actionsDiv = document.getElementById('processing-actions');
|
||||
if (actionsDiv) {
|
||||
actionsDiv.style.display = 'block';
|
||||
console.log('Processing actions div is now visible');
|
||||
} else {
|
||||
console.error('processing-actions div not found!');
|
||||
}
|
||||
}
|
||||
|
||||
const errorCount = data.errors || data.failed || 0;
|
||||
showDataConvertStatus(`Processing complete: ${processedCount} successful, ${errorCount} errors`,
|
||||
errorCount > 0 ? 'warning' : 'success');
|
||||
}
|
||||
|
||||
// Enhanced save function with better feedback
|
||||
async function saveGeocodedResults() {
|
||||
const successfulData = processingData.filter(item => item.geocode_success !== false && item.latitude && item.longitude);
|
||||
|
||||
if (successfulData.length === 0) {
|
||||
showDataConvertStatus('No successfully geocoded data to save', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Disable save button and show progress
|
||||
const saveBtn = document.getElementById('save-results-btn');
|
||||
const originalText = saveBtn.textContent;
|
||||
saveBtn.disabled = true;
|
||||
saveBtn.textContent = 'Adding to map...';
|
||||
|
||||
// Show saving status
|
||||
showDataConvertStatus(`Adding ${successfulData.length} locations to map...`, 'info');
|
||||
|
||||
let successCount = 0;
|
||||
let failedCount = 0;
|
||||
const errors = [];
|
||||
|
||||
try {
|
||||
// Use the bulk save endpoint instead of individual location creation
|
||||
// This is more efficient and avoids rate limiting issues
|
||||
const response = await fetch('/api/admin/data-convert/save-geocoded', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({ data: successfulData })
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
if (result.success) {
|
||||
successCount = result.results.success;
|
||||
failedCount = result.results.failed;
|
||||
|
||||
// Show detailed errors if any
|
||||
if (result.results.errors && result.results.errors.length > 0) {
|
||||
result.results.errors.forEach((error, index) => {
|
||||
errors.push(`Location ${index + 1}: ${error.error}`);
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`Bulk save completed: ${successCount} successful, ${failedCount} failed`);
|
||||
} else {
|
||||
throw new Error(result.error || 'Bulk save failed');
|
||||
}
|
||||
|
||||
// Show final results
|
||||
if (successCount > 0) {
|
||||
const message = `Successfully added ${successCount} locations to map.` +
|
||||
(failedCount > 0 ? ` ${failedCount} failed.` : '');
|
||||
showDataConvertStatus(message, failedCount > 0 ? 'warning' : 'success');
|
||||
|
||||
// Update UI to show completion
|
||||
saveBtn.textContent = `Added ${successCount} locations!`;
|
||||
setTimeout(() => {
|
||||
saveBtn.style.display = 'none';
|
||||
document.getElementById('new-upload-btn').style.display = 'inline-block';
|
||||
}, 3000);
|
||||
|
||||
} else {
|
||||
throw new Error('Failed to add any locations to the map');
|
||||
}
|
||||
|
||||
// Log errors if any
|
||||
if (errors.length > 0) {
|
||||
console.error('Import errors:', errors);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Save error:', error);
|
||||
showDataConvertStatus('Failed to add data to map: ' + error.message, 'error');
|
||||
saveBtn.disabled = false;
|
||||
saveBtn.textContent = originalText;
|
||||
}
|
||||
}
|
||||
|
||||
// Utility function to escape HTML
|
||||
function escapeHtml(text) {
|
||||
const map = {
|
||||
'&': '&',
|
||||
'<': '<',
|
||||
'>': '>',
|
||||
'"': '"',
|
||||
"'": '''
|
||||
};
|
||||
return text ? text.replace(/[&<>"']/g, function(m) { return map[m]; }) : '';
|
||||
}
|
||||
|
||||
// Clear upload and reset form
|
||||
function clearUpload() {
|
||||
const fileInput = document.getElementById('csv-file-input');
|
||||
const fileInfo = document.getElementById('file-info');
|
||||
const processBtn = document.getElementById('process-csv-btn');
|
||||
|
||||
if (fileInput) fileInput.value = '';
|
||||
if (fileInfo) fileInfo.style.display = 'none';
|
||||
if (processBtn) processBtn.disabled = true;
|
||||
}
|
||||
|
||||
// Reset to upload state
|
||||
function resetToUpload() {
|
||||
console.log('Resetting to upload state');
|
||||
|
||||
const uploadSection = document.getElementById('upload-section');
|
||||
const processingSection = document.getElementById('processing-section');
|
||||
const actionsDiv = document.getElementById('processing-actions');
|
||||
const resultsPreview = document.getElementById('results-preview');
|
||||
const saveBtn = document.getElementById('save-results-btn');
|
||||
const newUploadBtn = document.getElementById('new-upload-btn');
|
||||
|
||||
if (uploadSection) uploadSection.style.display = 'block';
|
||||
if (processingSection) processingSection.style.display = 'none';
|
||||
if (actionsDiv) actionsDiv.style.display = 'none';
|
||||
if (resultsPreview) resultsPreview.style.display = 'none';
|
||||
|
||||
// Reset buttons
|
||||
if (saveBtn) {
|
||||
saveBtn.style.display = 'inline-block';
|
||||
saveBtn.disabled = false;
|
||||
saveBtn.textContent = 'Add Data to Map';
|
||||
}
|
||||
if (newUploadBtn) {
|
||||
newUploadBtn.style.display = 'none';
|
||||
}
|
||||
|
||||
// Clear any existing data
|
||||
processingData = [];
|
||||
if (markers && markers.length > 0) {
|
||||
markers.forEach(marker => {
|
||||
if (resultsMap && marker) {
|
||||
try {
|
||||
resultsMap.removeLayer(marker);
|
||||
} catch (e) {
|
||||
console.warn('Error removing marker:', e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
markers = [];
|
||||
|
||||
// Reset results table
|
||||
const tbody = document.getElementById('results-tbody');
|
||||
if (tbody) {
|
||||
tbody.innerHTML = '';
|
||||
}
|
||||
|
||||
// Reset progress
|
||||
const progressFill = document.getElementById('progress-bar-fill');
|
||||
const progressCurrent = document.getElementById('progress-current');
|
||||
const progressTotal = document.getElementById('progress-total');
|
||||
const currentAddress = document.getElementById('current-address');
|
||||
|
||||
if (progressFill) progressFill.style.width = '0%';
|
||||
if (progressCurrent) progressCurrent.textContent = '0';
|
||||
if (progressTotal) progressTotal.textContent = '0';
|
||||
if (currentAddress) currentAddress.textContent = '';
|
||||
|
||||
// Destroy existing map if it exists
|
||||
if (resultsMap) {
|
||||
try {
|
||||
resultsMap.remove();
|
||||
resultsMap = null;
|
||||
} catch (e) {
|
||||
console.warn('Error removing map:', e);
|
||||
}
|
||||
}
|
||||
|
||||
// Reset form
|
||||
clearUpload();
|
||||
}
|
||||
Reference in New Issue
Block a user