Large update to geo-coding functions in order to support better matching of street addresses. Added premium mapbox option
This commit is contained in:
@@ -75,14 +75,71 @@ class DataConvertController {
|
||||
// Process each address with progress updates
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
const row = results[i];
|
||||
const address = row.address || row.Address || row.ADDRESS;
|
||||
|
||||
// Extract address - with better validation
|
||||
const addressField = row.address || row.Address || row.ADDRESS ||
|
||||
row.street_address || row['Street Address'] ||
|
||||
row.full_address || row['Full Address'];
|
||||
|
||||
// Extract unit number if available
|
||||
const unitField = row.unit || row.Unit || row.UNIT ||
|
||||
row.unit_number || row['Unit Number'] || row.unit_no;
|
||||
|
||||
if (!addressField || addressField.trim() === '') {
|
||||
logger.warn(`Row ${i + 1}: Empty or missing address field`);
|
||||
|
||||
const errorRow = {
|
||||
...row,
|
||||
latitude: '',
|
||||
longitude: '',
|
||||
'Geo-Location': '',
|
||||
geocoded_address: '',
|
||||
geocode_success: false,
|
||||
geocode_status: 'FAILED',
|
||||
geocode_error: 'Missing address field',
|
||||
csv_filename: originalFilename,
|
||||
row_number: i + 1
|
||||
};
|
||||
|
||||
allResults.push(errorRow);
|
||||
errors.push({
|
||||
index: i,
|
||||
address: 'No address provided',
|
||||
error: 'Missing address field'
|
||||
});
|
||||
|
||||
// Send progress update
|
||||
res.write(`data: ${JSON.stringify({
|
||||
type: 'progress',
|
||||
current: i + 1,
|
||||
total: total,
|
||||
currentAddress: 'No address - skipping',
|
||||
status: 'failed'
|
||||
})}\n\n`);
|
||||
res.flush && res.flush();
|
||||
|
||||
continue; // Skip to next row
|
||||
}
|
||||
|
||||
// Send progress update
|
||||
// Construct full address with unit if available
|
||||
let address = addressField.trim();
|
||||
if (unitField && unitField.toString().trim()) {
|
||||
const unit = unitField.toString().trim();
|
||||
// Add unit prefix if it doesn't already exist
|
||||
if (!unit.toLowerCase().startsWith('unit') &&
|
||||
!unit.toLowerCase().startsWith('apt') &&
|
||||
!unit.toLowerCase().startsWith('#')) {
|
||||
address = `Unit ${unit}, ${address}`;
|
||||
} else {
|
||||
address = `${unit}, ${address}`;
|
||||
}
|
||||
} // Send progress update
|
||||
res.write(`data: ${JSON.stringify({
|
||||
type: 'progress',
|
||||
current: i + 1,
|
||||
total: total,
|
||||
address: address
|
||||
currentAddress: address,
|
||||
status: 'processing'
|
||||
})}\n\n`);
|
||||
res.flush && res.flush();
|
||||
|
||||
@@ -93,6 +150,11 @@ class DataConvertController {
|
||||
const geocodeResult = await forwardGeocode(address);
|
||||
|
||||
if (geocodeResult && geocodeResult.coordinates) {
|
||||
// Check if result is malformed
|
||||
const isMalformed = geocodeResult.validation && geocodeResult.validation.isMalformed;
|
||||
const confidence = geocodeResult.validation ? geocodeResult.validation.confidence : 100;
|
||||
const warnings = geocodeResult.validation ? geocodeResult.validation.warnings : [];
|
||||
|
||||
const processedRow = {
|
||||
...row,
|
||||
latitude: geocodeResult.coordinates.lat,
|
||||
@@ -100,30 +162,38 @@ class DataConvertController {
|
||||
'Geo-Location': `${geocodeResult.coordinates.lat};${geocodeResult.coordinates.lng}`,
|
||||
geocoded_address: geocodeResult.formattedAddress || address,
|
||||
geocode_success: true,
|
||||
geocode_status: 'SUCCESS',
|
||||
geocode_status: isMalformed ? 'WARNING' : 'SUCCESS',
|
||||
geocode_error: '',
|
||||
csv_filename: originalFilename // Include filename for notes
|
||||
confidence_score: confidence,
|
||||
warnings: warnings.join('; '),
|
||||
is_malformed: isMalformed,
|
||||
provider: geocodeResult.provider || 'Unknown',
|
||||
csv_filename: originalFilename,
|
||||
row_number: i + 1
|
||||
};
|
||||
|
||||
processedData.push(processedRow);
|
||||
allResults.push(processedRow); // Add to full results for report
|
||||
allResults.push(processedRow);
|
||||
|
||||
// Send success update
|
||||
// Send success update with status
|
||||
const successMessage = {
|
||||
type: 'geocoded',
|
||||
data: processedRow,
|
||||
index: i
|
||||
index: i,
|
||||
status: isMalformed ? 'warning' : 'success',
|
||||
confidence: confidence,
|
||||
warnings: warnings
|
||||
};
|
||||
const successJson = JSON.stringify(successMessage);
|
||||
logger.debug(`Sending geocoded update: ${successJson.length} chars`);
|
||||
logger.info(`Successfully geocoded: ${address} (Confidence: ${confidence}%)`);
|
||||
res.write(`data: ${successJson}\n\n`);
|
||||
res.flush && res.flush(); // Ensure data is sent immediately
|
||||
res.flush && res.flush();
|
||||
} else {
|
||||
throw new Error('Geocoding failed - no coordinates returned');
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logger.error(`Failed to geocode address: ${address}`, error);
|
||||
logger.error(`Failed to geocode address: ${address}`, error.message);
|
||||
|
||||
// Create error row with original data plus error info
|
||||
const errorRow = {
|
||||
@@ -135,10 +205,14 @@ class DataConvertController {
|
||||
geocode_success: false,
|
||||
geocode_status: 'FAILED',
|
||||
geocode_error: error.message,
|
||||
csv_filename: originalFilename
|
||||
confidence_score: 0,
|
||||
warnings: '',
|
||||
is_malformed: false,
|
||||
csv_filename: originalFilename,
|
||||
row_number: i + 1
|
||||
};
|
||||
|
||||
allResults.push(errorRow); // Add to full results for report
|
||||
allResults.push(errorRow);
|
||||
|
||||
const errorData = {
|
||||
index: i,
|
||||
@@ -163,14 +237,21 @@ class DataConvertController {
|
||||
}
|
||||
|
||||
// Store processing results for report generation
|
||||
const successful = processedData.filter(r => r.geocode_status === 'SUCCESS').length;
|
||||
const warnings = processedData.filter(r => r.geocode_status === 'WARNING').length;
|
||||
const failed = errors.length;
|
||||
const malformed = processedData.filter(r => r.is_malformed).length;
|
||||
|
||||
processingResults.set(sessionId, {
|
||||
filename: originalFilename,
|
||||
timestamp: new Date().toISOString(),
|
||||
allResults: allResults,
|
||||
summary: {
|
||||
total: total,
|
||||
successful: processedData.length,
|
||||
failed: errors.length
|
||||
successful: successful,
|
||||
warnings: warnings,
|
||||
failed: failed,
|
||||
malformed: malformed
|
||||
}
|
||||
});
|
||||
|
||||
@@ -178,7 +259,10 @@ class DataConvertController {
|
||||
const completeMessage = {
|
||||
type: 'complete',
|
||||
processed: processedData.length,
|
||||
successful: successful,
|
||||
warnings: warnings,
|
||||
errors: errors.length,
|
||||
malformed: malformed,
|
||||
total: total,
|
||||
sessionId: sessionId // Include session ID for report download
|
||||
};
|
||||
@@ -382,18 +466,18 @@ class DataConvertController {
|
||||
const results = processingResults.get(sessionId);
|
||||
const { filename, timestamp, allResults, summary } = results;
|
||||
|
||||
// Convert results to CSV format
|
||||
const csvContent = this.generateReportCSV(allResults, filename, timestamp, summary);
|
||||
// Generate comprehensive report content
|
||||
const reportContent = this.generateComprehensiveReport(allResults, filename, timestamp, summary);
|
||||
|
||||
// Set headers for CSV download
|
||||
const reportFilename = `geocoding-report-${sessionId}.csv`;
|
||||
res.setHeader('Content-Type', 'text/csv');
|
||||
// Set headers for text download
|
||||
const reportFilename = `geocoding-report-${sessionId}.txt`;
|
||||
res.setHeader('Content-Type', 'text/plain');
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${reportFilename}"`);
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
|
||||
logger.info(`Generating report for session ${sessionId}: ${allResults.length} records`);
|
||||
logger.info(`Generating comprehensive report for session ${sessionId}: ${allResults.length} records`);
|
||||
|
||||
res.send(csvContent);
|
||||
res.send(reportContent);
|
||||
|
||||
// Clean up stored results after download (optional)
|
||||
setTimeout(() => {
|
||||
@@ -410,6 +494,101 @@ class DataConvertController {
|
||||
}
|
||||
}
|
||||
|
||||
// Generate comprehensive text report
|
||||
generateComprehensiveReport(results, originalFilename, timestamp, summary) {
|
||||
let report = `Geocoding Processing Report\n`;
|
||||
report += `Generated: ${timestamp}\n`;
|
||||
report += `Original File: ${originalFilename}\n`;
|
||||
report += `================================\n\n`;
|
||||
|
||||
report += `Summary:\n`;
|
||||
report += `- Total Addresses: ${summary.total}\n`;
|
||||
report += `- Successfully Geocoded: ${summary.successful}\n`;
|
||||
report += `- Warnings (Low Confidence): ${summary.warnings}\n`;
|
||||
report += `- Failed: ${summary.failed}\n`;
|
||||
report += `- Potentially Malformed: ${summary.malformed}\n\n`;
|
||||
|
||||
// Section for malformed addresses requiring review
|
||||
const malformedResults = results.filter(r => r.is_malformed);
|
||||
if (malformedResults.length > 0) {
|
||||
report += `ADDRESSES REQUIRING REVIEW (Potentially Malformed):\n`;
|
||||
report += `================================================\n`;
|
||||
malformedResults.forEach((result, index) => {
|
||||
const originalAddress = result.address || result.Address || result.ADDRESS || 'N/A';
|
||||
report += `\n${index + 1}. Original: ${originalAddress}\n`;
|
||||
report += ` Result: ${result.geocoded_address || 'N/A'}\n`;
|
||||
report += ` Confidence: ${result.confidence_score || 0}%\n`;
|
||||
if (result.warnings) {
|
||||
report += ` Warnings: ${result.warnings}\n`;
|
||||
}
|
||||
report += ` Coordinates: ${result.latitude || 'N/A'}, ${result.longitude || 'N/A'}\n`;
|
||||
report += ` Row: ${result.row_number}\n`;
|
||||
});
|
||||
report += `\n`;
|
||||
}
|
||||
|
||||
// Failed addresses section
|
||||
const failedResults = results.filter(r => r.geocode_status === 'FAILED');
|
||||
if (failedResults.length > 0) {
|
||||
report += `FAILED GEOCODING ATTEMPTS:\n`;
|
||||
report += `========================\n`;
|
||||
failedResults.forEach((result, index) => {
|
||||
const originalAddress = result.address || result.Address || result.ADDRESS || 'N/A';
|
||||
report += `\n${index + 1}. Address: ${originalAddress}\n`;
|
||||
report += ` Error: ${result.geocode_error}\n`;
|
||||
report += ` Row: ${result.row_number}\n`;
|
||||
});
|
||||
report += `\n`;
|
||||
}
|
||||
|
||||
// Successful geocoding with low confidence
|
||||
const lowConfidenceResults = results.filter(r =>
|
||||
r.geocode_status === 'SUCCESS' &&
|
||||
r.confidence_score &&
|
||||
r.confidence_score < 75
|
||||
);
|
||||
if (lowConfidenceResults.length > 0) {
|
||||
report += `LOW CONFIDENCE SUCCESSFUL GEOCODING:\n`;
|
||||
report += `==================================\n`;
|
||||
lowConfidenceResults.forEach((result, index) => {
|
||||
const originalAddress = result.address || result.Address || result.ADDRESS || 'N/A';
|
||||
report += `\n${index + 1}. Original: ${originalAddress}\n`;
|
||||
report += ` Result: ${result.geocoded_address}\n`;
|
||||
report += ` Confidence: ${result.confidence_score}%\n`;
|
||||
if (result.warnings) {
|
||||
report += ` Warnings: ${result.warnings}\n`;
|
||||
}
|
||||
report += ` Row: ${result.row_number}\n`;
|
||||
});
|
||||
report += `\n`;
|
||||
}
|
||||
|
||||
// Summary statistics
|
||||
report += `DETAILED STATISTICS:\n`;
|
||||
report += `==================\n`;
|
||||
report += `Success Rate: ${((summary.successful / summary.total) * 100).toFixed(1)}%\n`;
|
||||
report += `Warning Rate: ${((summary.warnings / summary.total) * 100).toFixed(1)}%\n`;
|
||||
report += `Failure Rate: ${((summary.failed / summary.total) * 100).toFixed(1)}%\n`;
|
||||
report += `Malformed Rate: ${((summary.malformed / summary.total) * 100).toFixed(1)}%\n\n`;
|
||||
|
||||
// Recommendations
|
||||
report += `RECOMMENDATIONS:\n`;
|
||||
report += `===============\n`;
|
||||
if (summary.malformed > 0) {
|
||||
report += `- Review ${summary.malformed} addresses marked as potentially malformed\n`;
|
||||
}
|
||||
if (summary.failed > 0) {
|
||||
report += `- Check ${summary.failed} failed addresses for formatting issues\n`;
|
||||
}
|
||||
if (summary.warnings > 0) {
|
||||
report += `- Verify ${summary.warnings} low confidence results manually\n`;
|
||||
}
|
||||
report += `- Consider using more specific address formats for better results\n`;
|
||||
report += `- Ensure addresses include proper directional indicators (NW, SW, etc.)\n`;
|
||||
|
||||
return report;
|
||||
}
|
||||
|
||||
// Generate CSV content for the report
|
||||
generateReportCSV(allResults, originalFilename, timestamp, summary) {
|
||||
if (!allResults || allResults.length === 0) {
|
||||
|
||||
Reference in New Issue
Block a user