download data import csv report
This commit is contained in:
@@ -5,12 +5,16 @@ const { forwardGeocode } = require('../services/geocoding');
|
||||
const logger = require('../utils/logger');
|
||||
const config = require('../config');
|
||||
|
||||
// In-memory storage for processing results (in production, use Redis or database)
|
||||
const processingResults = new Map();
|
||||
|
||||
class DataConvertController {
|
||||
constructor() {
|
||||
// Bind methods to preserve 'this' context
|
||||
this.processCSV = this.processCSV.bind(this);
|
||||
this.parseCSV = this.parseCSV.bind(this);
|
||||
this.saveGeocodedData = this.saveGeocodedData.bind(this);
|
||||
this.downloadReport = this.downloadReport.bind(this);
|
||||
}
|
||||
|
||||
// Process CSV upload and geocode addresses with SSE progress updates
|
||||
@@ -25,6 +29,7 @@ class DataConvertController {
|
||||
|
||||
// Store the filename for later use in notes
|
||||
const originalFilename = req.file.originalname;
|
||||
const sessionId = Date.now().toString(); // Simple session ID for storing results
|
||||
|
||||
// Set up SSE headers
|
||||
res.writeHead(200, {
|
||||
@@ -63,6 +68,7 @@ class DataConvertController {
|
||||
|
||||
// Process all addresses
|
||||
const processedData = [];
|
||||
const allResults = []; // Store ALL results for report generation
|
||||
const errors = [];
|
||||
const total = results.length;
|
||||
|
||||
@@ -94,10 +100,13 @@ class DataConvertController {
|
||||
'Geo-Location': `${geocodeResult.coordinates.lat};${geocodeResult.coordinates.lng}`,
|
||||
geocoded_address: geocodeResult.formattedAddress || address,
|
||||
geocode_success: true,
|
||||
geocode_status: 'SUCCESS',
|
||||
geocode_error: '',
|
||||
csv_filename: originalFilename // Include filename for notes
|
||||
};
|
||||
|
||||
processedData.push(processedRow);
|
||||
allResults.push(processedRow); // Add to full results for report
|
||||
|
||||
// Send success update
|
||||
const successMessage = {
|
||||
@@ -115,6 +124,22 @@ class DataConvertController {
|
||||
|
||||
} catch (error) {
|
||||
logger.error(`Failed to geocode address: ${address}`, error);
|
||||
|
||||
// Create error row with original data plus error info
|
||||
const errorRow = {
|
||||
...row,
|
||||
latitude: '',
|
||||
longitude: '',
|
||||
'Geo-Location': '',
|
||||
geocoded_address: '',
|
||||
geocode_success: false,
|
||||
geocode_status: 'FAILED',
|
||||
geocode_error: error.message,
|
||||
csv_filename: originalFilename
|
||||
};
|
||||
|
||||
allResults.push(errorRow); // Add to full results for report
|
||||
|
||||
const errorData = {
|
||||
index: i,
|
||||
address: address,
|
||||
@@ -137,12 +162,25 @@ class DataConvertController {
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
}
|
||||
|
||||
// Store processing results for report generation
|
||||
processingResults.set(sessionId, {
|
||||
filename: originalFilename,
|
||||
timestamp: new Date().toISOString(),
|
||||
allResults: allResults,
|
||||
summary: {
|
||||
total: total,
|
||||
successful: processedData.length,
|
||||
failed: errors.length
|
||||
}
|
||||
});
|
||||
|
||||
// Send completion
|
||||
const completeMessage = {
|
||||
type: 'complete',
|
||||
processed: processedData.length,
|
||||
errors: errors.length,
|
||||
total: total
|
||||
total: total,
|
||||
sessionId: sessionId // Include session ID for report download
|
||||
};
|
||||
const completeJson = JSON.stringify(completeMessage);
|
||||
logger.info(`Sending completion message: ${completeJson.length} chars`);
|
||||
@@ -302,6 +340,112 @@ class DataConvertController {
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Generate and download processing report
|
||||
async downloadReport(req, res) {
|
||||
try {
|
||||
const { sessionId } = req.params;
|
||||
|
||||
if (!sessionId || !processingResults.has(sessionId)) {
|
||||
return res.status(404).json({
|
||||
success: false,
|
||||
error: 'Processing results not found or expired'
|
||||
});
|
||||
}
|
||||
|
||||
const results = processingResults.get(sessionId);
|
||||
const { filename, timestamp, allResults, summary } = results;
|
||||
|
||||
// Convert results to CSV format
|
||||
const csvContent = this.generateReportCSV(allResults, filename, timestamp, summary);
|
||||
|
||||
// Set headers for CSV download
|
||||
const reportFilename = `geocoding-report-${sessionId}.csv`;
|
||||
res.setHeader('Content-Type', 'text/csv');
|
||||
res.setHeader('Content-Disposition', `attachment; filename="${reportFilename}"`);
|
||||
res.setHeader('Cache-Control', 'no-cache');
|
||||
|
||||
logger.info(`Generating report for session ${sessionId}: ${allResults.length} records`);
|
||||
|
||||
res.send(csvContent);
|
||||
|
||||
// Clean up stored results after download (optional)
|
||||
setTimeout(() => {
|
||||
processingResults.delete(sessionId);
|
||||
logger.info(`Cleaned up processing results for session ${sessionId}`);
|
||||
}, 60000); // Delete after 1 minute
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Download report error:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: 'Failed to generate report'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Generate CSV content for the report
|
||||
generateReportCSV(allResults, originalFilename, timestamp, summary) {
|
||||
if (!allResults || allResults.length === 0) {
|
||||
return 'No data available for report generation';
|
||||
}
|
||||
|
||||
// Get all unique field names from the results
|
||||
const allFields = new Set();
|
||||
allResults.forEach(row => {
|
||||
Object.keys(row).forEach(field => allFields.add(field));
|
||||
});
|
||||
|
||||
// Define the header order - put important fields first
|
||||
const priorityHeaders = [
|
||||
'geocode_status', 'geocode_error', 'address', 'Address',
|
||||
'geocoded_address', 'latitude', 'longitude', 'Geo-Location'
|
||||
];
|
||||
|
||||
const otherHeaders = Array.from(allFields).filter(field =>
|
||||
!priorityHeaders.includes(field) &&
|
||||
!['geocode_success', 'csv_filename'].includes(field)
|
||||
).sort();
|
||||
|
||||
const headers = [...priorityHeaders.filter(h => allFields.has(h)), ...otherHeaders];
|
||||
|
||||
// Generate CSV header with metadata
|
||||
let csvContent = `# Geocoding Processing Report\n`;
|
||||
csvContent += `# Original File: ${originalFilename}\n`;
|
||||
csvContent += `# Processed: ${timestamp}\n`;
|
||||
csvContent += `# Total Records: ${summary.total}\n`;
|
||||
csvContent += `# Successful: ${summary.successful}\n`;
|
||||
csvContent += `# Failed: ${summary.failed}\n`;
|
||||
csvContent += `# \n`;
|
||||
|
||||
// Add CSV headers
|
||||
csvContent += headers.map(header => this.escapeCSVField(header)).join(',') + '\n';
|
||||
|
||||
// Add data rows
|
||||
allResults.forEach(row => {
|
||||
const values = headers.map(header => {
|
||||
const value = row[header];
|
||||
return this.escapeCSVField(value !== undefined && value !== null ? String(value) : '');
|
||||
});
|
||||
csvContent += values.join(',') + '\n';
|
||||
});
|
||||
|
||||
return csvContent;
|
||||
}
|
||||
|
||||
// Escape CSV fields properly
|
||||
escapeCSVField(field) {
|
||||
if (field === null || field === undefined) return '';
|
||||
|
||||
const stringField = String(field);
|
||||
|
||||
// If field contains comma, quote, or newline, wrap in quotes and escape quotes
|
||||
if (stringField.includes(',') || stringField.includes('"') || stringField.includes('\n') || stringField.includes('\r')) {
|
||||
return '"' + stringField.replace(/"/g, '""') + '"';
|
||||
}
|
||||
|
||||
return stringField;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = new DataConvertController();
|
||||
|
||||
Reference in New Issue
Block a user