New update for the geo-coding system to include system to automatically scan the nocodb locations to build geo-locations
This commit is contained in:
@@ -1255,6 +1255,116 @@
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Scan & Geocode Existing Records -->
|
||||
<div class="scan-geocode-container" style="margin-top: 30px;">
|
||||
<div class="info-box">
|
||||
<h3>🔍 Scan & Geocode Database</h3>
|
||||
<p>Scan your existing database for records that are missing location data and automatically geocode them.</p>
|
||||
|
||||
<div class="scan-info">
|
||||
<h4>What this does:</h4>
|
||||
<ul>
|
||||
<li>Scans all records in your database</li>
|
||||
<li>Finds records with addresses but no geo-location data</li>
|
||||
<li>Automatically geocodes those addresses using the same multi-provider system</li>
|
||||
<li>Updates records with coordinates, confidence scores, and provider information</li>
|
||||
<li>Provides detailed progress tracking and error reporting</li>
|
||||
</ul>
|
||||
|
||||
<div class="scan-warning">
|
||||
<h4>⚠️ Important Notes:</h4>
|
||||
<ul>
|
||||
<li>This will modify existing records in your database</li>
|
||||
<li>Only processes records that have an address but no coordinates</li>
|
||||
<li>Rate limited to be respectful to geocoding APIs (0.5 seconds between requests)</li>
|
||||
<li>You can download a detailed report when completed</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="scan-actions">
|
||||
<button type="button" class="btn btn-primary" id="scan-geocode-btn">
|
||||
🔍 Start Database Scan & Geocode
|
||||
</button>
|
||||
<button type="button" class="btn btn-secondary" id="cancel-scan-btn" style="display: none;">
|
||||
⏹️ Cancel Scan
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="scan-processing-section" id="scan-processing-section" style="display: none;">
|
||||
<h3>Database Scan Progress</h3>
|
||||
|
||||
<div class="scan-phase" id="scan-phase">
|
||||
<h4>Phase 1: Database Scan</h4>
|
||||
<p id="scan-status">Scanning database for records...</p>
|
||||
</div>
|
||||
|
||||
<div class="progress-bar-container" id="scan-progress-container" style="display: none;">
|
||||
<div class="progress-bar">
|
||||
<div class="progress-bar-fill" id="scan-progress-bar-fill"></div>
|
||||
</div>
|
||||
<p class="progress-text">
|
||||
<span id="scan-progress-current">0</span> /
|
||||
<span id="scan-progress-total">0</span> addresses geocoded
|
||||
</p>
|
||||
</div>
|
||||
|
||||
<div class="processing-status" id="scan-processing-status">
|
||||
<p id="scan-current-address"></p>
|
||||
</div>
|
||||
|
||||
<div class="scan-summary" id="scan-summary" style="display: none;">
|
||||
<h4>Scan Summary</h4>
|
||||
<div class="summary-stats">
|
||||
<div class="stat-item">
|
||||
<span class="stat-label">Total Records:</span>
|
||||
<span class="stat-value" id="total-records">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-label">Need Geocoding:</span>
|
||||
<span class="stat-value" id="need-geocoding">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-label">Successfully Geocoded:</span>
|
||||
<span class="stat-value" id="successfully-geocoded">0</span>
|
||||
</div>
|
||||
<div class="stat-item">
|
||||
<span class="stat-label">Failed:</span>
|
||||
<span class="stat-value" id="failed-geocoded">0</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="scan-results-preview" id="scan-results-preview" style="display: none;">
|
||||
<h4>Recent Results</h4>
|
||||
<div class="scan-results-table-container">
|
||||
<table class="results-table" id="scan-results-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Status</th>
|
||||
<th>Address</th>
|
||||
<th>Coordinates</th>
|
||||
<th>Confidence</th>
|
||||
<th>Provider</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="scan-results-tbody"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="scan-actions-completed" id="scan-actions-completed" style="display: none;">
|
||||
<button type="button" class="btn btn-primary" id="download-scan-report-btn">
|
||||
📄 Download Detailed Report
|
||||
</button>
|
||||
<button type="button" class="btn btn-secondary" id="new-scan-btn">
|
||||
🔍 Start New Scan
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Email Lists (Listmonk) Section -->
|
||||
|
||||
@@ -189,6 +189,29 @@ function setupDataConvertEventListeners() {
|
||||
newUploadBtn.addEventListener('click', resetToUpload);
|
||||
}
|
||||
|
||||
// Scan & Geocode Database button
|
||||
const scanGeocodeBtn = document.getElementById('scan-geocode-btn');
|
||||
const cancelScanBtn = document.getElementById('cancel-scan-btn');
|
||||
const downloadScanReportBtn = document.getElementById('download-scan-report-btn');
|
||||
const newScanBtn = document.getElementById('new-scan-btn');
|
||||
|
||||
if (scanGeocodeBtn) {
|
||||
scanGeocodeBtn.addEventListener('click', startDatabaseScan);
|
||||
console.log('Scan geocode button event listener attached');
|
||||
}
|
||||
|
||||
if (cancelScanBtn) {
|
||||
cancelScanBtn.addEventListener('click', cancelDatabaseScan);
|
||||
}
|
||||
|
||||
if (downloadScanReportBtn) {
|
||||
downloadScanReportBtn.addEventListener('click', downloadScanReport);
|
||||
}
|
||||
|
||||
if (newScanBtn) {
|
||||
newScanBtn.addEventListener('click', resetScanInterface);
|
||||
}
|
||||
|
||||
// Mark as initialized
|
||||
eventListenersInitialized = true;
|
||||
console.log('Data convert event listeners initialized successfully');
|
||||
@@ -937,3 +960,296 @@ async function downloadProcessingReport() {
|
||||
downloadBtn.disabled = false;
|
||||
}
|
||||
}
|
||||
|
||||
// === DATABASE SCAN & GEOCODE FUNCTIONALITY ===
|
||||
|
||||
async function startDatabaseScan() {
|
||||
try {
|
||||
const scanBtn = document.getElementById('scan-geocode-btn');
|
||||
const cancelBtn = document.getElementById('cancel-scan-btn');
|
||||
const processingSection = document.getElementById('scan-processing-section');
|
||||
const scanPhase = document.getElementById('scan-phase');
|
||||
const scanStatus = document.getElementById('scan-status');
|
||||
const progressContainer = document.getElementById('scan-progress-container');
|
||||
const summary = document.getElementById('scan-summary');
|
||||
const resultsPreview = document.getElementById('scan-results-preview');
|
||||
const actionsCompleted = document.getElementById('scan-actions-completed');
|
||||
|
||||
// Show processing UI and hide scan button
|
||||
if (scanBtn) scanBtn.style.display = 'none';
|
||||
if (cancelBtn) cancelBtn.style.display = 'inline-block';
|
||||
if (processingSection) processingSection.style.display = 'block';
|
||||
|
||||
// Reset UI state
|
||||
if (progressContainer) progressContainer.style.display = 'none';
|
||||
if (summary) summary.style.display = 'none';
|
||||
if (resultsPreview) resultsPreview.style.display = 'none';
|
||||
if (actionsCompleted) actionsCompleted.style.display = 'none';
|
||||
|
||||
// Clear previous results
|
||||
const scanResultsTable = document.getElementById('scan-results-tbody');
|
||||
if (scanResultsTable) scanResultsTable.innerHTML = '';
|
||||
|
||||
console.log('Starting database scan and geocode...');
|
||||
|
||||
// Make POST request for Server-Sent Events
|
||||
const response = await fetch('/api/admin/data-convert/scan-and-geocode', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'Accept': 'text/event-stream'
|
||||
}
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status}: ${response.statusText}`);
|
||||
}
|
||||
|
||||
// Process the stream manually
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
|
||||
let scanSessionId = null;
|
||||
let scanResults = [];
|
||||
|
||||
try {
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
const chunk = decoder.decode(value);
|
||||
const lines = chunk.split('\n');
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('data: ') && line.trim() !== 'data: ') {
|
||||
try {
|
||||
const data = JSON.parse(line.substring(6));
|
||||
console.log('Scan event received:', data);
|
||||
|
||||
switch (data.type) {
|
||||
case 'status':
|
||||
if (scanStatus) scanStatus.textContent = data.message;
|
||||
if (data.sessionId) {
|
||||
scanSessionId = data.sessionId;
|
||||
console.log('Initial scan session ID set:', scanSessionId);
|
||||
}
|
||||
break;
|
||||
|
||||
case 'scanning':
|
||||
if (scanStatus) scanStatus.textContent = data.message;
|
||||
break;
|
||||
|
||||
case 'scan_complete':
|
||||
if (scanStatus) scanStatus.textContent = data.message;
|
||||
|
||||
// Update scan summary
|
||||
document.getElementById('total-records').textContent = data.total || 0;
|
||||
document.getElementById('need-geocoding').textContent = data.needingGeocode || 0;
|
||||
document.getElementById('successfully-geocoded').textContent = '0';
|
||||
document.getElementById('failed-geocoded').textContent = '0';
|
||||
|
||||
if (summary) summary.style.display = 'block';
|
||||
|
||||
if (data.needingGeocode > 0) {
|
||||
// Show progress bar and update phase
|
||||
if (progressContainer) progressContainer.style.display = 'block';
|
||||
if (scanPhase) {
|
||||
scanPhase.innerHTML = '<h4>Phase 2: Geocoding Addresses</h4>';
|
||||
}
|
||||
|
||||
// Update progress totals
|
||||
document.getElementById('scan-progress-total').textContent = data.needingGeocode;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'progress':
|
||||
// Update progress bar
|
||||
const progressPercent = (data.current / data.total) * 100;
|
||||
const progressBar = document.getElementById('scan-progress-bar-fill');
|
||||
if (progressBar) progressBar.style.width = `${progressPercent}%`;
|
||||
|
||||
document.getElementById('scan-progress-current').textContent = data.current;
|
||||
document.getElementById('scan-current-address').textContent = `Processing: ${data.currentAddress}`;
|
||||
break;
|
||||
|
||||
case 'geocoded':
|
||||
// Update success counter
|
||||
const successCount = parseInt(document.getElementById('successfully-geocoded').textContent) + 1;
|
||||
document.getElementById('successfully-geocoded').textContent = successCount;
|
||||
|
||||
// Add to results preview
|
||||
addScanResultToTable(data.data, data.status);
|
||||
scanResults.push(data.data);
|
||||
|
||||
// Show results preview if not already visible
|
||||
if (resultsPreview) resultsPreview.style.display = 'block';
|
||||
break;
|
||||
|
||||
case 'error':
|
||||
if (data.data) {
|
||||
// Update failed counter
|
||||
const failedCount = parseInt(document.getElementById('failed-geocoded').textContent) + 1;
|
||||
document.getElementById('failed-geocoded').textContent = failedCount;
|
||||
|
||||
// Add to results preview
|
||||
addScanResultToTable(data.data, 'error');
|
||||
scanResults.push(data.data);
|
||||
} else {
|
||||
// General error
|
||||
if (scanStatus) scanStatus.textContent = `Error: ${data.message}`;
|
||||
}
|
||||
break;
|
||||
|
||||
case 'complete':
|
||||
if (scanStatus) scanStatus.textContent = data.message;
|
||||
|
||||
// Show completed actions
|
||||
if (actionsCompleted) actionsCompleted.style.display = 'block';
|
||||
|
||||
// Store session ID for report download from the completion message
|
||||
const finalSessionId = data.sessionId || data.results?.sessionId || scanSessionId;
|
||||
if (finalSessionId) {
|
||||
const downloadBtn = document.getElementById('download-scan-report-btn');
|
||||
if (downloadBtn) {
|
||||
downloadBtn.dataset.sessionId = finalSessionId;
|
||||
console.log('Stored scan session ID for report download:', finalSessionId);
|
||||
}
|
||||
} else {
|
||||
console.warn('No session ID available for scan report download');
|
||||
}
|
||||
|
||||
// Hide cancel button
|
||||
if (cancelBtn) cancelBtn.style.display = 'none';
|
||||
|
||||
// Exit the loop when complete
|
||||
return;
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error parsing scan event:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Stream processing error:', error);
|
||||
if (scanStatus) scanStatus.textContent = 'Error processing stream';
|
||||
} finally {
|
||||
// Clean up reader
|
||||
reader.releaseLock();
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Error starting database scan:', error);
|
||||
alert('Failed to start database scan: ' + error.message);
|
||||
}
|
||||
}
|
||||
|
||||
function cancelDatabaseScan() {
|
||||
console.log('Cancelling database scan...');
|
||||
|
||||
// Note: With fetch streams, we can't easily cancel mid-stream
|
||||
// but we can reset the UI to let user know cancellation was requested
|
||||
|
||||
// Reset UI
|
||||
const scanBtn = document.getElementById('scan-geocode-btn');
|
||||
const cancelBtn = document.getElementById('cancel-scan-btn');
|
||||
const scanStatus = document.getElementById('scan-status');
|
||||
|
||||
if (scanBtn) scanBtn.style.display = 'inline-block';
|
||||
if (cancelBtn) cancelBtn.style.display = 'none';
|
||||
if (scanStatus) scanStatus.textContent = 'Scan cancelled by user.';
|
||||
}
|
||||
|
||||
function addScanResultToTable(result, status) {
|
||||
const tbody = document.getElementById('scan-results-tbody');
|
||||
if (!tbody) return;
|
||||
|
||||
// Limit table to last 10 results
|
||||
while (tbody.children.length >= 10) {
|
||||
tbody.removeChild(tbody.firstChild);
|
||||
}
|
||||
|
||||
const row = document.createElement('tr');
|
||||
row.className = `status-${status}`;
|
||||
|
||||
const statusIcon = status === 'success' ? '✅' :
|
||||
status === 'warning' ? '⚠️' : '❌';
|
||||
|
||||
const coordinates = result.latitude && result.longitude ?
|
||||
`${result.latitude}, ${result.longitude}` : 'Failed';
|
||||
|
||||
const confidence = result.confidence_score !== undefined ?
|
||||
`${result.confidence_score}%` : 'N/A';
|
||||
|
||||
row.innerHTML = `
|
||||
<td>${statusIcon} ${status.toUpperCase()}</td>
|
||||
<td>${result.address || 'Unknown'}</td>
|
||||
<td>${coordinates}</td>
|
||||
<td>${confidence}</td>
|
||||
<td>${result.provider || 'N/A'}</td>
|
||||
`;
|
||||
|
||||
tbody.appendChild(row);
|
||||
}
|
||||
|
||||
async function downloadScanReport() {
|
||||
const downloadBtn = document.getElementById('download-scan-report-btn');
|
||||
const sessionId = downloadBtn?.dataset.sessionId;
|
||||
|
||||
if (!sessionId) {
|
||||
alert('No processing session available for report generation');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`/api/admin/data-convert/download-report/${sessionId}`);
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
// Get the filename from the response headers
|
||||
const filename = response.headers.get('content-disposition')
|
||||
?.split('filename=')[1]
|
||||
?.replace(/"/g, '') || `scan-geocoding-report-${sessionId}.txt`;
|
||||
|
||||
// Create blob and download
|
||||
const blob = await response.blob();
|
||||
const url = window.URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url;
|
||||
a.download = filename;
|
||||
document.body.appendChild(a);
|
||||
a.click();
|
||||
window.URL.revokeObjectURL(url);
|
||||
document.body.removeChild(a);
|
||||
|
||||
// Show success message in scan status area
|
||||
const scanStatus = document.getElementById('scan-status');
|
||||
if (scanStatus) {
|
||||
scanStatus.innerHTML = '✅ Report downloaded successfully';
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Download scan report error:', error);
|
||||
// Show error message in scan status area
|
||||
const scanStatus = document.getElementById('scan-status');
|
||||
if (scanStatus) {
|
||||
scanStatus.innerHTML = `❌ Failed to download report: ${error.message}`;
|
||||
} else {
|
||||
alert(`Failed to download report: ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function resetScanInterface() {
|
||||
// Reset to initial state
|
||||
const scanBtn = document.getElementById('scan-geocode-btn');
|
||||
const cancelBtn = document.getElementById('cancel-scan-btn');
|
||||
const processingSection = document.getElementById('scan-processing-section');
|
||||
|
||||
if (scanBtn) scanBtn.style.display = 'inline-block';
|
||||
if (cancelBtn) cancelBtn.style.display = 'none';
|
||||
if (processingSection) processingSection.style.display = 'none';
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user