Data converter
This commit is contained in:
307
map/app/controllers/dataConvertController.js
Normal file
307
map/app/controllers/dataConvertController.js
Normal file
@@ -0,0 +1,307 @@
|
||||
const csv = require('csv-parse');
|
||||
const { Readable } = require('stream');
|
||||
const nocodbService = require('../services/nocodb');
|
||||
const { forwardGeocode } = require('../services/geocoding');
|
||||
const logger = require('../utils/logger');
|
||||
const config = require('../config');
|
||||
|
||||
class DataConvertController {
|
||||
constructor() {
|
||||
// Bind methods to preserve 'this' context
|
||||
this.processCSV = this.processCSV.bind(this);
|
||||
this.parseCSV = this.parseCSV.bind(this);
|
||||
this.saveGeocodedData = this.saveGeocodedData.bind(this);
|
||||
}
|
||||
|
||||
// Process CSV upload and geocode addresses with SSE progress updates
|
||||
async processCSV(req, res) {
|
||||
try {
|
||||
if (!req.file) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: 'No file uploaded'
|
||||
});
|
||||
}
|
||||
|
||||
// Store the filename for later use in notes
|
||||
const originalFilename = req.file.originalname;
|
||||
|
||||
// Set up SSE headers
|
||||
res.writeHead(200, {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
'X-Accel-Buffering': 'no' // Disable Nginx buffering
|
||||
});
|
||||
|
||||
// Parse CSV
|
||||
const results = await this.parseCSV(req.file.buffer);
|
||||
|
||||
if (!results || results.length === 0) {
|
||||
res.write(`data: ${JSON.stringify({ type: 'error', message: 'CSV file is empty or invalid' })}\n\n`);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
// Validate required address field
|
||||
const hasAddressField = results[0].hasOwnProperty('address') ||
|
||||
results[0].hasOwnProperty('Address') ||
|
||||
results[0].hasOwnProperty('ADDRESS');
|
||||
|
||||
if (!hasAddressField) {
|
||||
res.write(`data: ${JSON.stringify({ type: 'error', message: 'CSV must contain an "address" column' })}\n\n`);
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
// Send initial progress
|
||||
res.write(`data: ${JSON.stringify({
|
||||
type: 'start',
|
||||
total: results.length
|
||||
})}\n\n`);
|
||||
res.flush && res.flush();
|
||||
|
||||
// Process all addresses
|
||||
const processedData = [];
|
||||
const errors = [];
|
||||
const total = results.length;
|
||||
|
||||
// Process each address with progress updates
|
||||
for (let i = 0; i < results.length; i++) {
|
||||
const row = results[i];
|
||||
const address = row.address || row.Address || row.ADDRESS;
|
||||
|
||||
// Send progress update
|
||||
res.write(`data: ${JSON.stringify({
|
||||
type: 'progress',
|
||||
current: i + 1,
|
||||
total: total,
|
||||
address: address
|
||||
})}\n\n`);
|
||||
res.flush && res.flush();
|
||||
|
||||
try {
|
||||
logger.info(`Geocoding ${i + 1}/${total}: ${address}`);
|
||||
|
||||
// Geocode the address
|
||||
const geocodeResult = await forwardGeocode(address);
|
||||
|
||||
if (geocodeResult && geocodeResult.coordinates) {
|
||||
const processedRow = {
|
||||
...row,
|
||||
latitude: geocodeResult.coordinates.lat,
|
||||
longitude: geocodeResult.coordinates.lng,
|
||||
'Geo-Location': `${geocodeResult.coordinates.lat};${geocodeResult.coordinates.lng}`,
|
||||
geocoded_address: geocodeResult.formattedAddress || address,
|
||||
geocode_success: true,
|
||||
csv_filename: originalFilename // Include filename for notes
|
||||
};
|
||||
|
||||
processedData.push(processedRow);
|
||||
|
||||
// Send success update
|
||||
const successMessage = {
|
||||
type: 'geocoded',
|
||||
data: processedRow,
|
||||
index: i
|
||||
};
|
||||
const successJson = JSON.stringify(successMessage);
|
||||
logger.debug(`Sending geocoded update: ${successJson.length} chars`);
|
||||
res.write(`data: ${successJson}\n\n`);
|
||||
res.flush && res.flush(); // Ensure data is sent immediately
|
||||
} else {
|
||||
throw new Error('Geocoding failed - no coordinates returned');
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
logger.error(`Failed to geocode address: ${address}`, error);
|
||||
const errorData = {
|
||||
index: i,
|
||||
address: address,
|
||||
error: error.message
|
||||
};
|
||||
errors.push(errorData);
|
||||
|
||||
// Send error update
|
||||
const errorMessage = {
|
||||
type: 'error',
|
||||
data: errorData
|
||||
};
|
||||
const errorJson = JSON.stringify(errorMessage);
|
||||
logger.debug(`Sending error update: ${errorJson.length} chars`);
|
||||
res.write(`data: ${errorJson}\n\n`);
|
||||
res.flush && res.flush(); // Ensure data is sent immediately
|
||||
}
|
||||
|
||||
// Add delay to avoid rate limiting
|
||||
await new Promise(resolve => setTimeout(resolve, 2000));
|
||||
}
|
||||
|
||||
// Send completion
|
||||
const completeMessage = {
|
||||
type: 'complete',
|
||||
processed: processedData.length,
|
||||
errors: errors.length,
|
||||
total: total
|
||||
};
|
||||
const completeJson = JSON.stringify(completeMessage);
|
||||
logger.info(`Sending completion message: ${completeJson.length} chars`);
|
||||
res.write(`data: ${completeJson}\n\n`);
|
||||
res.flush && res.flush(); // Ensure data is sent immediately
|
||||
|
||||
res.end();
|
||||
|
||||
} catch (error) {
|
||||
logger.error('CSV processing error:', error);
|
||||
res.write(`data: ${JSON.stringify({
|
||||
type: 'fatal_error',
|
||||
message: 'Failed to process CSV file',
|
||||
error: error.message
|
||||
})}\n\n`);
|
||||
res.end();
|
||||
}
|
||||
}
|
||||
|
||||
// Parse CSV buffer into array of objects
|
||||
async parseCSV(buffer) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const results = [];
|
||||
const stream = Readable.from(buffer);
|
||||
|
||||
stream
|
||||
.pipe(csv.parse({
|
||||
columns: true,
|
||||
skip_empty_lines: true,
|
||||
trim: true
|
||||
}))
|
||||
.on('data', (data) => results.push(data))
|
||||
.on('error', reject)
|
||||
.on('end', () => resolve(results));
|
||||
});
|
||||
}
|
||||
|
||||
// Enhanced save method that transforms data to match locations table structure
|
||||
async saveGeocodedData(req, res) {
|
||||
try {
|
||||
const { data } = req.body;
|
||||
|
||||
if (!data || !Array.isArray(data)) {
|
||||
return res.status(400).json({
|
||||
success: false,
|
||||
error: 'Invalid data format'
|
||||
});
|
||||
}
|
||||
|
||||
const results = {
|
||||
success: 0,
|
||||
failed: 0,
|
||||
errors: []
|
||||
};
|
||||
|
||||
// Process each location
|
||||
for (const location of data) {
|
||||
try {
|
||||
// Transform to match locations table structure
|
||||
const locationData = {
|
||||
'Geo-Location': location['Geo-Location'],
|
||||
latitude: parseFloat(location.latitude),
|
||||
longitude: parseFloat(location.longitude),
|
||||
Address: location.geocoded_address || location.address || location.Address,
|
||||
created_by_user: req.session.userEmail || 'csv_import',
|
||||
last_updated_by_user: req.session.userEmail || 'csv_import'
|
||||
};
|
||||
|
||||
// Map CSV fields to NocoDB fields
|
||||
const fieldMapping = {
|
||||
'first name': 'First Name',
|
||||
'firstname': 'First Name',
|
||||
'first_name': 'First Name',
|
||||
'last name': 'Last Name',
|
||||
'lastname': 'Last Name',
|
||||
'last_name': 'Last Name',
|
||||
'email': 'Email',
|
||||
'phone': 'Phone',
|
||||
'unit': 'Unit Number',
|
||||
'unit number': 'Unit Number',
|
||||
'unit_number': 'Unit Number',
|
||||
'support level': 'Support Level',
|
||||
'support_level': 'Support Level',
|
||||
'sign': 'Sign',
|
||||
'sign size': 'Sign Size',
|
||||
'sign_size': 'Sign Size',
|
||||
'notes': 'Notes'
|
||||
};
|
||||
|
||||
// Process all fields from CSV
|
||||
Object.keys(location).forEach(key => {
|
||||
const lowerKey = key.toLowerCase();
|
||||
|
||||
// Skip already processed fields
|
||||
if (['latitude', 'longitude', 'geo-location', 'geocoded_address', 'geocode_success', 'address', 'csv_filename'].includes(lowerKey)) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if we have a mapping for this field
|
||||
if (fieldMapping[lowerKey]) {
|
||||
const targetField = fieldMapping[lowerKey];
|
||||
|
||||
// Special handling for certain fields
|
||||
if (targetField === 'Sign') {
|
||||
// Convert to boolean
|
||||
locationData[targetField] = ['true', 'yes', '1', 'y'].includes(String(location[key]).toLowerCase());
|
||||
} else if (targetField === 'Support Level') {
|
||||
// Ensure it's a string number 1-4
|
||||
const level = parseInt(location[key]);
|
||||
if (level >= 1 && level <= 4) {
|
||||
locationData[targetField] = String(level);
|
||||
}
|
||||
} else if (targetField === 'Notes') {
|
||||
// Append CSV filename info to existing notes
|
||||
const existingNotes = location[key] || '';
|
||||
const csvInfo = `Imported from CSV: ${location.csv_filename || 'unknown'}`;
|
||||
locationData[targetField] = existingNotes ?
|
||||
`${existingNotes} | ${csvInfo}` :
|
||||
csvInfo;
|
||||
} else {
|
||||
locationData[targetField] = location[key];
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// If no notes field was found in CSV, add the CSV import info
|
||||
if (!locationData['Notes']) {
|
||||
locationData['Notes'] = `Imported from CSV: ${location.csv_filename || 'unknown'}`;
|
||||
}
|
||||
|
||||
// Create location in NocoDB
|
||||
const result = await nocodbService.create(config.nocodb.tableId, locationData);
|
||||
results.success++;
|
||||
logger.debug(`Successfully saved location: ${locationData.Address}`);
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Failed to save location:', error);
|
||||
logger.error('Location data:', locationData);
|
||||
results.failed++;
|
||||
results.errors.push({
|
||||
address: location.address || location.Address || location.geocoded_address,
|
||||
error: error.message
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
res.json({
|
||||
success: true,
|
||||
results: results
|
||||
});
|
||||
|
||||
} catch (error) {
|
||||
logger.error('Save geocoded data error:', error);
|
||||
res.status(500).json({
|
||||
success: false,
|
||||
error: 'Failed to save locations'
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = new DataConvertController();
|
||||
6
map/app/package-lock.json
generated
6
map/app/package-lock.json
generated
@@ -12,6 +12,7 @@
|
||||
"axios": "^1.6.0",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"cors": "^2.8.5",
|
||||
"csv-parse": "^6.1.0",
|
||||
"dotenv": "^16.3.1",
|
||||
"express": "^4.18.2",
|
||||
"express-rate-limit": "^7.1.4",
|
||||
@@ -502,6 +503,11 @@
|
||||
"node": ">= 0.10"
|
||||
}
|
||||
},
|
||||
"node_modules/csv-parse": {
|
||||
"version": "6.1.0",
|
||||
"resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-6.1.0.tgz",
|
||||
"integrity": "sha512-CEE+jwpgLn+MmtCpVcPtiCZpVtB6Z2OKPTr34pycYYoL7sxdOkXDdQ4lRiw6ioC0q6BLqhc6cKweCVvral8yhw=="
|
||||
},
|
||||
"node_modules/debug": {
|
||||
"version": "2.6.9",
|
||||
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
|
||||
|
||||
@@ -21,6 +21,7 @@
|
||||
"axios": "^1.6.0",
|
||||
"cookie-parser": "^1.4.7",
|
||||
"cors": "^2.8.5",
|
||||
"csv-parse": "^6.1.0",
|
||||
"dotenv": "^16.3.1",
|
||||
"express": "^4.18.2",
|
||||
"express-rate-limit": "^7.1.4",
|
||||
|
||||
@@ -63,6 +63,10 @@
|
||||
<span class="nav-icon">👥</span>
|
||||
<span class="nav-text">Users</span>
|
||||
</a>
|
||||
<a href="#convert-data">
|
||||
<span class="nav-icon">📊</span>
|
||||
<span class="nav-text">Convert Data</span>
|
||||
</a>
|
||||
</nav>
|
||||
<div class="sidebar-footer">
|
||||
<div id="mobile-admin-info" class="mobile-admin-info mobile-only"></div>
|
||||
@@ -353,6 +357,191 @@
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
|
||||
<!-- Convert Data Section -->
|
||||
<section id="convert-data" class="admin-section" style="display: none;">
|
||||
<h2>Convert Data</h2>
|
||||
<p>Upload a CSV file containing addresses to geocode and import into the map.</p>
|
||||
|
||||
<div class="data-convert-container">
|
||||
<div class="upload-section" id="upload-section">
|
||||
<h3>CSV Upload</h3>
|
||||
<form id="csv-upload-form">
|
||||
<div class="upload-area" id="upload-area">
|
||||
<div class="upload-icon">📁</div>
|
||||
<p>Drag and drop your CSV file here or click to browse</p>
|
||||
<input type="file" id="csv-file-input" accept=".csv" style="display: none;">
|
||||
<button type="button" class="btn btn-primary" id="browse-btn">Choose File</button>
|
||||
</div>
|
||||
|
||||
<div class="file-info" id="file-info" style="display: none;">
|
||||
<p><strong>Selected file:</strong> <span id="file-name"></span></p>
|
||||
<p><strong>Size:</strong> <span id="file-size"></span></p>
|
||||
</div>
|
||||
|
||||
<div class="csv-requirements">
|
||||
<h4>CSV Requirements:</h4>
|
||||
<div class="requirements-section">
|
||||
<h5>Required Column:</h5>
|
||||
<ul>
|
||||
<li><strong>address</strong> - The street address to geocode (case-insensitive)</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="requirements-section">
|
||||
<h5>Optional Columns (any of these names will work):</h5>
|
||||
<div class="field-mapping-grid">
|
||||
<div class="field-group">
|
||||
<strong>First Name:</strong>
|
||||
<ul>
|
||||
<li>first name</li>
|
||||
<li>firstname</li>
|
||||
<li>first_name</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="field-group">
|
||||
<strong>Last Name:</strong>
|
||||
<ul>
|
||||
<li>last name</li>
|
||||
<li>lastname</li>
|
||||
<li>last_name</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="field-group">
|
||||
<strong>Email:</strong>
|
||||
<ul>
|
||||
<li>email</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="field-group">
|
||||
<strong>Phone:</strong>
|
||||
<ul>
|
||||
<li>phone</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="field-group">
|
||||
<strong>Unit Number:</strong>
|
||||
<ul>
|
||||
<li>unit</li>
|
||||
<li>unit number</li>
|
||||
<li>unit_number</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="field-group">
|
||||
<strong>Support Level (1-4):</strong>
|
||||
<ul>
|
||||
<li>support level</li>
|
||||
<li>support_level</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="field-group">
|
||||
<strong>Sign (true/false):</strong>
|
||||
<ul>
|
||||
<li>sign</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="field-group">
|
||||
<strong>Sign Size (Regular, Large, Unsure)</strong>
|
||||
<ul>
|
||||
<li>sign size</li>
|
||||
<li>sign_size</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="field-group">
|
||||
<strong>Notes:</strong>
|
||||
<ul>
|
||||
<li>notes</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="requirements-section">
|
||||
<h5>File Specifications:</h5>
|
||||
<ul>
|
||||
<li>Maximum file size: 10MB</li>
|
||||
<li>Column names are case-insensitive</li>
|
||||
<li>Extra columns will be ignored</li>
|
||||
</ul>
|
||||
</div>
|
||||
|
||||
<div class="requirements-section">
|
||||
<h5>Example CSV Format:</h5>
|
||||
<pre class="csv-example">address,first name,last name,email,phone,support level,notes
|
||||
"123 Main St, Edmonton, AB",John,Doe,john@example.com,780-555-0123,2,Interested in campaign
|
||||
"456 Oak Ave, Edmonton, AB",Jane,Smith,jane@example.com,780-555-0456,1,Strong supporter</pre>
|
||||
<p style="margin-top: 10px;">
|
||||
<a href="data:text/csv;charset=utf-8,address%2Cfirst%20name%2Clast%20name%2Cemail%2Cphone%2Csupport%20level%2Cnotes%0A%22123%20Main%20St%2C%20Edmonton%2C%20AB%22%2CJohn%2CDoe%2Cjohn%40example.com%2C780-555-0123%2C2%2CInterested%20in%20campaign%0A%22456%20Oak%20Ave%2C%20Edmonton%2C%20AB%22%2CJane%2CSmith%2Cjane%40example.com%2C780-555-0456%2C1%2CStrong%20supporter"
|
||||
download="sample-import.csv"
|
||||
class="btn btn-secondary btn-sm">
|
||||
📄 Download Sample CSV
|
||||
</a>
|
||||
</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-actions">
|
||||
<button type="submit" class="btn btn-primary" id="process-csv-btn" disabled>
|
||||
Process CSV
|
||||
</button>
|
||||
<button type="button" class="btn btn-secondary" id="clear-upload-btn">
|
||||
Clear
|
||||
</button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="processing-section" id="processing-section" style="display: none;">
|
||||
<h3>Processing Progress</h3>
|
||||
|
||||
<div class="progress-bar-container">
|
||||
<div class="progress-bar">
|
||||
<div class="progress-bar-fill" id="progress-bar-fill"></div>
|
||||
</div>
|
||||
<p class="progress-text"><span id="progress-current">0</span> / <span id="progress-total">0</span> addresses processed</p>
|
||||
</div>
|
||||
|
||||
<div class="processing-status" id="processing-status">
|
||||
<p id="current-address"></p>
|
||||
</div>
|
||||
|
||||
<div class="results-preview" id="results-preview" style="display: none;">
|
||||
<h4>Preview Results</h4>
|
||||
<div class="results-map" id="results-map"></div>
|
||||
<div class="results-table-container">
|
||||
<table class="results-table" id="results-table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Status</th>
|
||||
<th>Original Address</th>
|
||||
<th>Geocoded Address</th>
|
||||
<th>Coordinates</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="results-tbody"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="processing-actions" id="processing-actions" style="display: none;">
|
||||
<button type="button" class="btn btn-success" id="save-results-btn">
|
||||
Add Data to Map
|
||||
</button>
|
||||
<button type="button" class="btn btn-secondary" id="new-upload-btn">
|
||||
Upload New File
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</section>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -427,7 +616,9 @@
|
||||
<!-- Dashboard JavaScript -->
|
||||
<script src="js/dashboard.js"></script>
|
||||
|
||||
<!-- Data Convert JavaScript -->
|
||||
<!-- Admin JavaScript -->
|
||||
<script src="js/admin.js"></script>
|
||||
<script src="js/data-convert.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -643,6 +643,96 @@
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
/* Data Convert Styles */
|
||||
.data-convert-container {
|
||||
max-width: 1000px;
|
||||
}
|
||||
|
||||
.csv-requirements {
|
||||
background: #f8f9fa;
|
||||
border: 1px solid #dee2e6;
|
||||
border-radius: 8px;
|
||||
padding: 20px;
|
||||
margin: 20px 0;
|
||||
}
|
||||
|
||||
.csv-requirements h4 {
|
||||
color: #495057;
|
||||
margin-bottom: 15px;
|
||||
border-bottom: 2px solid #e9ecef;
|
||||
padding-bottom: 10px;
|
||||
}
|
||||
|
||||
.requirements-section {
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.requirements-section:last-child {
|
||||
margin-bottom: 0;
|
||||
}
|
||||
|
||||
.requirements-section h5 {
|
||||
color: #6c757d;
|
||||
margin-bottom: 10px;
|
||||
font-weight: 600;
|
||||
}
|
||||
|
||||
.field-mapping-grid {
|
||||
display: grid;
|
||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
||||
gap: 15px;
|
||||
margin-top: 10px;
|
||||
}
|
||||
|
||||
.field-group {
|
||||
background: white;
|
||||
border: 1px solid #e9ecef;
|
||||
border-radius: 6px;
|
||||
padding: 12px;
|
||||
}
|
||||
|
||||
.field-group strong {
|
||||
color: #495057;
|
||||
display: block;
|
||||
margin-bottom: 8px;
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.field-group ul {
|
||||
margin: 0;
|
||||
padding-left: 15px;
|
||||
list-style-type: disc;
|
||||
}
|
||||
|
||||
.field-group li {
|
||||
font-family: 'Courier New', monospace;
|
||||
font-size: 12px;
|
||||
color: #6c757d;
|
||||
margin-bottom: 3px;
|
||||
}
|
||||
|
||||
.csv-example {
|
||||
background: #2d3748;
|
||||
color: #e2e8f0;
|
||||
padding: 15px;
|
||||
border-radius: 6px;
|
||||
font-family: 'Courier New', monospace;
|
||||
font-size: 12px;
|
||||
overflow-x: auto;
|
||||
margin-top: 10px;
|
||||
white-space: pre;
|
||||
}
|
||||
|
||||
.requirements-section ul {
|
||||
margin: 10px 0;
|
||||
padding-left: 20px;
|
||||
}
|
||||
|
||||
.requirements-section > ul > li {
|
||||
margin-bottom: 5px;
|
||||
color: #495057;
|
||||
}
|
||||
|
||||
.user-form,
|
||||
.users-list {
|
||||
background: white;
|
||||
@@ -838,6 +928,20 @@
|
||||
box-sizing: border-box;
|
||||
}
|
||||
|
||||
/* Data Convert responsive styles */
|
||||
.field-mapping-grid {
|
||||
grid-template-columns: 1fr;
|
||||
}
|
||||
|
||||
.csv-example {
|
||||
font-size: 10px;
|
||||
padding: 10px;
|
||||
}
|
||||
|
||||
.csv-requirements {
|
||||
padding: 15px;
|
||||
}
|
||||
|
||||
.users-table {
|
||||
font-size: 14px; /* Match desktop font size for better readability */
|
||||
min-width: auto; /* Remove minimum width constraint on mobile */
|
||||
@@ -1864,3 +1968,239 @@
|
||||
gap: 25px;
|
||||
}
|
||||
}
|
||||
|
||||
/* Convert Data Styles */
|
||||
.data-convert-container {
|
||||
display: flex;
|
||||
flex-direction: column;
|
||||
gap: 2rem;
|
||||
}
|
||||
|
||||
.upload-section {
|
||||
background: white;
|
||||
border-radius: 8px;
|
||||
padding: 2rem;
|
||||
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.upload-area {
|
||||
border: 2px dashed #ddd;
|
||||
border-radius: 8px;
|
||||
padding: 3rem;
|
||||
text-align: center;
|
||||
cursor: pointer;
|
||||
transition: all 0.3s ease;
|
||||
}
|
||||
|
||||
.upload-area:hover {
|
||||
border-color: var(--primary-color);
|
||||
background-color: #f8f9fa;
|
||||
}
|
||||
|
||||
.upload-area.drag-over {
|
||||
border-color: var(--primary-color);
|
||||
background-color: #e3f2fd;
|
||||
}
|
||||
|
||||
.upload-icon {
|
||||
font-size: 48px;
|
||||
margin-bottom: 1rem;
|
||||
}
|
||||
|
||||
.file-info {
|
||||
margin-top: 1rem;
|
||||
padding: 1rem;
|
||||
background: #f8f9fa;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.csv-requirements {
|
||||
margin-top: 1.5rem;
|
||||
padding: 1rem;
|
||||
background: #fff3cd;
|
||||
border: 1px solid #ffeeba;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.csv-requirements h4 {
|
||||
margin: 0 0 0.5rem 0;
|
||||
color: #856404;
|
||||
}
|
||||
|
||||
.csv-requirements ul {
|
||||
margin: 0;
|
||||
padding-left: 1.5rem;
|
||||
}
|
||||
|
||||
.processing-section {
|
||||
background: white;
|
||||
border-radius: 8px;
|
||||
padding: 2rem;
|
||||
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.1);
|
||||
}
|
||||
|
||||
.progress-bar-container {
|
||||
margin: 1.5rem 0;
|
||||
}
|
||||
|
||||
.progress-bar {
|
||||
height: 24px;
|
||||
background: #e9ecef;
|
||||
border-radius: 12px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
.progress-bar-fill {
|
||||
height: 100%;
|
||||
background: linear-gradient(90deg, #4CAF50 0%, #45a049 100%);
|
||||
transition: width 0.3s ease;
|
||||
width: 0%;
|
||||
}
|
||||
|
||||
.progress-text {
|
||||
text-align: center;
|
||||
margin-top: 0.5rem;
|
||||
font-weight: 500;
|
||||
}
|
||||
|
||||
.processing-status {
|
||||
padding: 1rem;
|
||||
background: #f8f9fa;
|
||||
border-radius: 4px;
|
||||
text-align: center;
|
||||
margin-bottom: 1.5rem;
|
||||
animation: pulse 1.5s ease-in-out infinite;
|
||||
}
|
||||
|
||||
/* Pulsing effect for current address */
|
||||
@keyframes pulse {
|
||||
0% { opacity: 0.8; }
|
||||
50% { opacity: 1; }
|
||||
100% { opacity: 0.8; }
|
||||
}
|
||||
|
||||
/* Success row animation */
|
||||
.result-success {
|
||||
animation: slideIn 0.3s ease-out;
|
||||
}
|
||||
|
||||
.result-error {
|
||||
animation: slideIn 0.3s ease-out;
|
||||
}
|
||||
|
||||
@keyframes slideIn {
|
||||
from {
|
||||
transform: translateX(-20px);
|
||||
opacity: 0;
|
||||
}
|
||||
to {
|
||||
transform: translateX(0);
|
||||
opacity: 1;
|
||||
}
|
||||
}
|
||||
|
||||
.results-preview {
|
||||
margin-top: 2rem;
|
||||
}
|
||||
|
||||
.results-map {
|
||||
height: 400px;
|
||||
margin-bottom: 1.5rem;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.results-table-container {
|
||||
max-height: 400px;
|
||||
overflow-y: auto;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 4px;
|
||||
}
|
||||
|
||||
.results-table {
|
||||
width: 100%;
|
||||
border-collapse: collapse;
|
||||
}
|
||||
|
||||
.results-table th {
|
||||
background: #f8f9fa;
|
||||
padding: 0.75rem;
|
||||
text-align: left;
|
||||
font-weight: 600;
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 10;
|
||||
}
|
||||
|
||||
.results-table td {
|
||||
padding: 0.75rem;
|
||||
border-bottom: 1px solid #e9ecef;
|
||||
}
|
||||
|
||||
.result-success {
|
||||
background: #d4edda;
|
||||
}
|
||||
|
||||
.result-error {
|
||||
background: #f8d7da;
|
||||
}
|
||||
|
||||
.status-icon {
|
||||
display: inline-block;
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
line-height: 20px;
|
||||
text-align: center;
|
||||
border-radius: 50%;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.status-icon.success {
|
||||
background: #28a745;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.status-icon.error {
|
||||
background: #dc3545;
|
||||
color: white;
|
||||
}
|
||||
|
||||
.error-message {
|
||||
color: #721c24;
|
||||
font-style: italic;
|
||||
}
|
||||
|
||||
.processing-actions {
|
||||
display: flex;
|
||||
gap: 1rem;
|
||||
justify-content: center;
|
||||
margin-top: 2rem;
|
||||
}
|
||||
|
||||
/* Mobile responsiveness for Convert Data */
|
||||
@media (max-width: 768px) {
|
||||
.upload-area {
|
||||
padding: 2rem;
|
||||
}
|
||||
|
||||
.upload-icon {
|
||||
font-size: 36px;
|
||||
}
|
||||
|
||||
.results-map {
|
||||
height: 300px;
|
||||
}
|
||||
|
||||
.results-table {
|
||||
font-size: 14px;
|
||||
}
|
||||
|
||||
.results-table th,
|
||||
.results-table td {
|
||||
padding: 0.5rem;
|
||||
}
|
||||
|
||||
.processing-actions {
|
||||
flex-direction: column;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -413,6 +413,20 @@ function setupNavigation() {
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// If switching to convert-data section, ensure event listeners are set up
|
||||
if (targetId === 'convert-data') {
|
||||
console.log('Convert Data section activated');
|
||||
// Initialize data convert functionality if available
|
||||
setTimeout(() => {
|
||||
if (typeof window.setupDataConvertEventListeners === 'function') {
|
||||
console.log('Setting up data convert event listeners...');
|
||||
window.setupDataConvertEventListeners();
|
||||
} else {
|
||||
console.warn('setupDataConvertEventListeners function not available');
|
||||
}
|
||||
}, 100); // Small delay to ensure DOM is ready
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
585
map/app/public/js/data-convert.js
Normal file
585
map/app/public/js/data-convert.js
Normal file
@@ -0,0 +1,585 @@
|
||||
let processingData = [];
|
||||
let resultsMap = null;
|
||||
let markers = [];
|
||||
|
||||
// Utility function to show status messages
|
||||
function showDataConvertStatus(message, type = 'info') {
|
||||
// Try to use the global showStatus from admin.js if available
|
||||
if (typeof window.showStatus === 'function') {
|
||||
return window.showStatus(message, type);
|
||||
}
|
||||
|
||||
// Fallback to console
|
||||
console.log(`[${type.toUpperCase()}] ${message}`);
|
||||
|
||||
// Try to display in status container if it exists
|
||||
const statusContainer = document.getElementById('status-container');
|
||||
if (statusContainer) {
|
||||
const statusEl = document.createElement('div');
|
||||
statusEl.className = `status-message status-${type}`;
|
||||
statusEl.textContent = message;
|
||||
statusContainer.appendChild(statusEl);
|
||||
|
||||
// Auto-remove after 5 seconds
|
||||
setTimeout(() => {
|
||||
if (statusEl.parentNode) {
|
||||
statusEl.parentNode.removeChild(statusEl);
|
||||
}
|
||||
}, 5000);
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize when DOM is loaded
|
||||
document.addEventListener('DOMContentLoaded', () => {
|
||||
console.log('Data convert JS loaded');
|
||||
// Don't auto-initialize, wait for section to be activated
|
||||
|
||||
// Make setupDataConvertEventListeners available globally for admin.js
|
||||
window.setupDataConvertEventListeners = setupDataConvertEventListeners;
|
||||
});
|
||||
|
||||
function setupDataConvertEventListeners() {
|
||||
console.log('Setting up data convert event listeners...');
|
||||
|
||||
const fileInput = document.getElementById('csv-file-input');
|
||||
const browseBtn = document.getElementById('browse-btn');
|
||||
const uploadArea = document.getElementById('upload-area');
|
||||
const uploadForm = document.getElementById('csv-upload-form');
|
||||
const clearBtn = document.getElementById('clear-upload-btn');
|
||||
const saveResultsBtn = document.getElementById('save-results-btn');
|
||||
const newUploadBtn = document.getElementById('new-upload-btn');
|
||||
|
||||
console.log('Elements found:', {
|
||||
fileInput: !!fileInput,
|
||||
browseBtn: !!browseBtn,
|
||||
uploadArea: !!uploadArea,
|
||||
uploadForm: !!uploadForm,
|
||||
clearBtn: !!clearBtn,
|
||||
saveResultsBtn: !!saveResultsBtn,
|
||||
newUploadBtn: !!newUploadBtn
|
||||
});
|
||||
|
||||
// File input change
|
||||
if (fileInput) {
|
||||
fileInput.addEventListener('change', handleFileSelect);
|
||||
}
|
||||
|
||||
// Browse button
|
||||
if (browseBtn) {
|
||||
browseBtn.addEventListener('click', () => {
|
||||
console.log('Browse button clicked');
|
||||
fileInput?.click();
|
||||
});
|
||||
}
|
||||
|
||||
// Drag and drop
|
||||
if (uploadArea) {
|
||||
uploadArea.addEventListener('dragover', handleDragOver);
|
||||
uploadArea.addEventListener('dragleave', handleDragLeave);
|
||||
uploadArea.addEventListener('drop', handleDrop);
|
||||
uploadArea.addEventListener('click', () => fileInput?.click());
|
||||
}
|
||||
|
||||
// Form submission
|
||||
if (uploadForm) {
|
||||
uploadForm.addEventListener('submit', handleCSVUpload);
|
||||
}
|
||||
|
||||
// Clear button
|
||||
if (clearBtn) {
|
||||
clearBtn.addEventListener('click', clearUpload);
|
||||
}
|
||||
|
||||
// Save results button - ADD DATA TO MAP
|
||||
if (saveResultsBtn) {
|
||||
saveResultsBtn.addEventListener('click', saveGeocodedResults);
|
||||
console.log('Save results button event listener attached');
|
||||
}
|
||||
|
||||
// New upload button
|
||||
if (newUploadBtn) {
|
||||
newUploadBtn.addEventListener('click', resetToUpload);
|
||||
}
|
||||
}
|
||||
|
||||
function handleFileSelect(e) {
|
||||
const file = e.target.files[0];
|
||||
if (file) {
|
||||
displayFileInfo(file);
|
||||
}
|
||||
}
|
||||
|
||||
function handleDragOver(e) {
|
||||
e.preventDefault();
|
||||
e.currentTarget.classList.add('drag-over');
|
||||
}
|
||||
|
||||
function handleDragLeave(e) {
|
||||
e.preventDefault();
|
||||
e.currentTarget.classList.remove('drag-over');
|
||||
}
|
||||
|
||||
function handleDrop(e) {
|
||||
e.preventDefault();
|
||||
e.currentTarget.classList.remove('drag-over');
|
||||
|
||||
const files = e.dataTransfer.files;
|
||||
if (files.length > 0) {
|
||||
const file = files[0];
|
||||
if (file.type === 'text/csv' || file.name.endsWith('.csv')) {
|
||||
document.getElementById('csv-file-input').files = files;
|
||||
displayFileInfo(file);
|
||||
} else {
|
||||
showDataConvertStatus('Please upload a CSV file', 'error');
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function displayFileInfo(file) {
|
||||
document.getElementById('file-info').style.display = 'block';
|
||||
document.getElementById('file-name').textContent = file.name;
|
||||
document.getElementById('file-size').textContent = formatFileSize(file.size);
|
||||
document.getElementById('process-csv-btn').disabled = false;
|
||||
}
|
||||
|
||||
function formatFileSize(bytes) {
|
||||
if (bytes === 0) return '0 Bytes';
|
||||
const k = 1024;
|
||||
const sizes = ['Bytes', 'KB', 'MB'];
|
||||
const i = Math.floor(Math.log(bytes) / Math.log(k));
|
||||
return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
|
||||
}
|
||||
|
||||
async function handleCSVUpload(e) {
|
||||
e.preventDefault();
|
||||
|
||||
const fileInput = document.getElementById('csv-file-input');
|
||||
const file = fileInput?.files[0];
|
||||
|
||||
if (!file) {
|
||||
showDataConvertStatus('Please select a CSV file', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Reset processing data
|
||||
processingData = [];
|
||||
|
||||
// Show processing section
|
||||
const uploadSection = document.getElementById('upload-section');
|
||||
const processingSection = document.getElementById('processing-section');
|
||||
|
||||
if (!uploadSection || !processingSection) {
|
||||
console.error('Required DOM elements not found');
|
||||
showDataConvertStatus('Interface error: required elements not found', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
uploadSection.style.display = 'none';
|
||||
processingSection.style.display = 'block';
|
||||
|
||||
// Initialize results map
|
||||
initializeResultsMap();
|
||||
|
||||
// Create form data
|
||||
const formData = new FormData();
|
||||
formData.append('csvFile', file);
|
||||
|
||||
try {
|
||||
// Send the file and handle SSE response
|
||||
const response = await fetch('/api/admin/data-convert/process-csv', {
|
||||
method: 'POST',
|
||||
body: formData
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
// Read the response as a stream for SSE
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = ''; // Buffer to accumulate partial data
|
||||
|
||||
while (true) {
|
||||
const { done, value } = await reader.read();
|
||||
if (done) break;
|
||||
|
||||
// Decode the chunk and add to buffer
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
|
||||
// Split buffer by lines and process complete lines
|
||||
const lines = buffer.split('\n');
|
||||
|
||||
// Keep the last potentially incomplete line in the buffer
|
||||
buffer = lines.pop() || '';
|
||||
|
||||
for (const line of lines) {
|
||||
if (line.startsWith('data: ')) {
|
||||
try {
|
||||
const jsonData = line.substring(6).trim(); // Remove 'data: ' prefix and trim
|
||||
if (jsonData && jsonData !== '') {
|
||||
const data = JSON.parse(jsonData);
|
||||
handleProcessingUpdate(data);
|
||||
}
|
||||
} catch (parseError) {
|
||||
console.warn('Failed to parse SSE data:', parseError);
|
||||
console.warn('Problematic line:', line);
|
||||
console.warn('JSON data:', line.substring(6));
|
||||
}
|
||||
} else if (line.trim() === '') {
|
||||
// Empty line, ignore
|
||||
continue;
|
||||
} else if (line.trim() !== '' && !line.startsWith('event:') && !line.startsWith('id:')) {
|
||||
// Unexpected line format
|
||||
console.warn('Unexpected SSE line format:', line);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process any remaining data in buffer
|
||||
if (buffer.trim()) {
|
||||
const line = buffer;
|
||||
if (line.startsWith('data: ')) {
|
||||
try {
|
||||
const jsonData = line.substring(6).trim();
|
||||
if (jsonData && jsonData !== '') {
|
||||
const data = JSON.parse(jsonData);
|
||||
handleProcessingUpdate(data);
|
||||
}
|
||||
} catch (parseError) {
|
||||
console.warn('Failed to parse final SSE data:', parseError);
|
||||
console.warn('Final buffer content:', buffer);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('CSV processing error:', error);
|
||||
showDataConvertStatus(error.message || 'Failed to process CSV', 'error');
|
||||
resetToUpload();
|
||||
}
|
||||
}
|
||||
|
||||
// Enhanced processing update handler
|
||||
function handleProcessingUpdate(data) {
|
||||
console.log('Processing update:', data);
|
||||
|
||||
// Validate data structure
|
||||
if (!data || typeof data !== 'object') {
|
||||
console.warn('Invalid data received:', data);
|
||||
return;
|
||||
}
|
||||
|
||||
switch (data.type) {
|
||||
case 'start':
|
||||
updateProgress(0, data.total);
|
||||
document.getElementById('current-address').textContent = 'Starting geocoding process...';
|
||||
break;
|
||||
|
||||
case 'progress':
|
||||
updateProgress(data.current, data.total);
|
||||
document.getElementById('current-address').textContent = `Processing: ${data.address}`;
|
||||
break;
|
||||
|
||||
case 'geocoded':
|
||||
// Mark as successful and add to processing data
|
||||
const successData = { ...data.data, geocode_success: true };
|
||||
processingData.push(successData);
|
||||
addResultToTable(successData, 'success');
|
||||
addMarkerToMap(successData);
|
||||
updateProgress(data.index + 1, data.total);
|
||||
break;
|
||||
|
||||
case 'error':
|
||||
// Mark as failed and add to processing data
|
||||
const errorData = { ...data.data, geocode_success: false };
|
||||
processingData.push(errorData);
|
||||
addResultToTable(errorData, 'error');
|
||||
break;
|
||||
|
||||
case 'complete':
|
||||
console.log('Received complete event:', data);
|
||||
onProcessingComplete(data);
|
||||
break;
|
||||
|
||||
case 'fatal_error':
|
||||
showDataConvertStatus(data.message, 'error');
|
||||
resetToUpload();
|
||||
break;
|
||||
|
||||
default:
|
||||
console.warn('Unknown data type received:', data.type);
|
||||
}
|
||||
}
|
||||
|
||||
function updateProgress(current, total) {
|
||||
const percentage = (current / total) * 100;
|
||||
document.getElementById('progress-bar-fill').style.width = percentage + '%';
|
||||
document.getElementById('progress-current').textContent = current;
|
||||
document.getElementById('progress-total').textContent = total;
|
||||
}
|
||||
|
||||
function initializeResultsMap() {
|
||||
const mapContainer = document.getElementById('results-map');
|
||||
if (!mapContainer) return;
|
||||
|
||||
// Initialize map
|
||||
resultsMap = L.map('results-map').setView([53.5461, -113.4938], 11);
|
||||
|
||||
L.tileLayer('https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png', {
|
||||
attribution: '© OpenStreetMap contributors'
|
||||
}).addTo(resultsMap);
|
||||
|
||||
document.getElementById('results-preview').style.display = 'block';
|
||||
|
||||
// Fix map sizing
|
||||
setTimeout(() => {
|
||||
resultsMap.invalidateSize();
|
||||
}, 100);
|
||||
}
|
||||
|
||||
function addResultToTable(data, status) {
|
||||
const tbody = document.getElementById('results-tbody');
|
||||
const row = document.createElement('tr');
|
||||
row.className = status === 'success' ? 'result-success' : 'result-error';
|
||||
|
||||
if (status === 'success') {
|
||||
row.innerHTML = `
|
||||
<td><span class="status-icon success">✓</span></td>
|
||||
<td>${escapeHtml(data.address || data.Address || '')}</td>
|
||||
<td>${escapeHtml(data.geocoded_address || '')}</td>
|
||||
<td>${data.latitude.toFixed(6)}, ${data.longitude.toFixed(6)}</td>
|
||||
`;
|
||||
} else {
|
||||
row.innerHTML = `
|
||||
<td><span class="status-icon error">✗</span></td>
|
||||
<td>${escapeHtml(data.address || '')}</td>
|
||||
<td colspan="2" class="error-message">${escapeHtml(data.error || 'Geocoding failed')}</td>
|
||||
`;
|
||||
}
|
||||
|
||||
tbody.appendChild(row);
|
||||
}
|
||||
|
||||
function addMarkerToMap(data) {
|
||||
if (!resultsMap || !data.latitude || !data.longitude) return;
|
||||
|
||||
const marker = L.marker([data.latitude, data.longitude])
|
||||
.bindPopup(`
|
||||
<strong>${escapeHtml(data.geocoded_address || data.address)}</strong><br>
|
||||
${data.latitude.toFixed(6)}, ${data.longitude.toFixed(6)}
|
||||
`);
|
||||
|
||||
marker.addTo(resultsMap);
|
||||
markers.push(marker);
|
||||
|
||||
// Adjust map bounds to show all markers
|
||||
if (markers.length > 0) {
|
||||
const group = new L.featureGroup(markers);
|
||||
resultsMap.fitBounds(group.getBounds().pad(0.1));
|
||||
}
|
||||
}
|
||||
|
||||
function onProcessingComplete(data) {
|
||||
console.log('Processing complete called with data:', data);
|
||||
|
||||
document.getElementById('current-address').textContent =
|
||||
`Complete! Processed ${data.processed || data.success || 0} addresses successfully, ${data.errors || data.failed || 0} errors.`;
|
||||
|
||||
const processedCount = data.processed || data.success || processingData.filter(item => item.geocode_success !== false).length;
|
||||
|
||||
if (processedCount > 0) {
|
||||
console.log('Showing processing actions for', processedCount, 'successful items');
|
||||
const actionsDiv = document.getElementById('processing-actions');
|
||||
if (actionsDiv) {
|
||||
actionsDiv.style.display = 'block';
|
||||
console.log('Processing actions div is now visible');
|
||||
} else {
|
||||
console.error('processing-actions div not found!');
|
||||
}
|
||||
}
|
||||
|
||||
const errorCount = data.errors || data.failed || 0;
|
||||
showDataConvertStatus(`Processing complete: ${processedCount} successful, ${errorCount} errors`,
|
||||
errorCount > 0 ? 'warning' : 'success');
|
||||
}
|
||||
|
||||
// Enhanced save function with better feedback
|
||||
async function saveGeocodedResults() {
|
||||
const successfulData = processingData.filter(item => item.geocode_success !== false && item.latitude && item.longitude);
|
||||
|
||||
if (successfulData.length === 0) {
|
||||
showDataConvertStatus('No successfully geocoded data to save', 'error');
|
||||
return;
|
||||
}
|
||||
|
||||
// Disable save button and show progress
|
||||
const saveBtn = document.getElementById('save-results-btn');
|
||||
const originalText = saveBtn.textContent;
|
||||
saveBtn.disabled = true;
|
||||
saveBtn.textContent = 'Adding to map...';
|
||||
|
||||
// Show saving status
|
||||
showDataConvertStatus(`Adding ${successfulData.length} locations to map...`, 'info');
|
||||
|
||||
let successCount = 0;
|
||||
let failedCount = 0;
|
||||
const errors = [];
|
||||
|
||||
try {
|
||||
// Use the bulk save endpoint instead of individual location creation
|
||||
// This is more efficient and avoids rate limiting issues
|
||||
const response = await fetch('/api/admin/data-convert/save-geocoded', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json'
|
||||
},
|
||||
body: JSON.stringify({ data: successfulData })
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP error! status: ${response.status}`);
|
||||
}
|
||||
|
||||
const result = await response.json();
|
||||
|
||||
if (result.success) {
|
||||
successCount = result.results.success;
|
||||
failedCount = result.results.failed;
|
||||
|
||||
// Show detailed errors if any
|
||||
if (result.results.errors && result.results.errors.length > 0) {
|
||||
result.results.errors.forEach((error, index) => {
|
||||
errors.push(`Location ${index + 1}: ${error.error}`);
|
||||
});
|
||||
}
|
||||
|
||||
console.log(`Bulk save completed: ${successCount} successful, ${failedCount} failed`);
|
||||
} else {
|
||||
throw new Error(result.error || 'Bulk save failed');
|
||||
}
|
||||
|
||||
// Show final results
|
||||
if (successCount > 0) {
|
||||
const message = `Successfully added ${successCount} locations to map.` +
|
||||
(failedCount > 0 ? ` ${failedCount} failed.` : '');
|
||||
showDataConvertStatus(message, failedCount > 0 ? 'warning' : 'success');
|
||||
|
||||
// Update UI to show completion
|
||||
saveBtn.textContent = `Added ${successCount} locations!`;
|
||||
setTimeout(() => {
|
||||
saveBtn.style.display = 'none';
|
||||
document.getElementById('new-upload-btn').style.display = 'inline-block';
|
||||
}, 3000);
|
||||
|
||||
} else {
|
||||
throw new Error('Failed to add any locations to the map');
|
||||
}
|
||||
|
||||
// Log errors if any
|
||||
if (errors.length > 0) {
|
||||
console.error('Import errors:', errors);
|
||||
}
|
||||
|
||||
} catch (error) {
|
||||
console.error('Save error:', error);
|
||||
showDataConvertStatus('Failed to add data to map: ' + error.message, 'error');
|
||||
saveBtn.disabled = false;
|
||||
saveBtn.textContent = originalText;
|
||||
}
|
||||
}
|
||||
|
||||
// Utility function to escape HTML
|
||||
function escapeHtml(text) {
|
||||
const map = {
|
||||
'&': '&',
|
||||
'<': '<',
|
||||
'>': '>',
|
||||
'"': '"',
|
||||
"'": '''
|
||||
};
|
||||
return text ? text.replace(/[&<>"']/g, function(m) { return map[m]; }) : '';
|
||||
}
|
||||
|
||||
// Clear upload and reset form
|
||||
function clearUpload() {
|
||||
const fileInput = document.getElementById('csv-file-input');
|
||||
const fileInfo = document.getElementById('file-info');
|
||||
const processBtn = document.getElementById('process-csv-btn');
|
||||
|
||||
if (fileInput) fileInput.value = '';
|
||||
if (fileInfo) fileInfo.style.display = 'none';
|
||||
if (processBtn) processBtn.disabled = true;
|
||||
}
|
||||
|
||||
// Reset to upload state
|
||||
function resetToUpload() {
|
||||
console.log('Resetting to upload state');
|
||||
|
||||
const uploadSection = document.getElementById('upload-section');
|
||||
const processingSection = document.getElementById('processing-section');
|
||||
const actionsDiv = document.getElementById('processing-actions');
|
||||
const resultsPreview = document.getElementById('results-preview');
|
||||
const saveBtn = document.getElementById('save-results-btn');
|
||||
const newUploadBtn = document.getElementById('new-upload-btn');
|
||||
|
||||
if (uploadSection) uploadSection.style.display = 'block';
|
||||
if (processingSection) processingSection.style.display = 'none';
|
||||
if (actionsDiv) actionsDiv.style.display = 'none';
|
||||
if (resultsPreview) resultsPreview.style.display = 'none';
|
||||
|
||||
// Reset buttons
|
||||
if (saveBtn) {
|
||||
saveBtn.style.display = 'inline-block';
|
||||
saveBtn.disabled = false;
|
||||
saveBtn.textContent = 'Add Data to Map';
|
||||
}
|
||||
if (newUploadBtn) {
|
||||
newUploadBtn.style.display = 'none';
|
||||
}
|
||||
|
||||
// Clear any existing data
|
||||
processingData = [];
|
||||
if (markers && markers.length > 0) {
|
||||
markers.forEach(marker => {
|
||||
if (resultsMap && marker) {
|
||||
try {
|
||||
resultsMap.removeLayer(marker);
|
||||
} catch (e) {
|
||||
console.warn('Error removing marker:', e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
markers = [];
|
||||
|
||||
// Reset results table
|
||||
const tbody = document.getElementById('results-tbody');
|
||||
if (tbody) {
|
||||
tbody.innerHTML = '';
|
||||
}
|
||||
|
||||
// Reset progress
|
||||
const progressFill = document.getElementById('progress-bar-fill');
|
||||
const progressCurrent = document.getElementById('progress-current');
|
||||
const progressTotal = document.getElementById('progress-total');
|
||||
const currentAddress = document.getElementById('current-address');
|
||||
|
||||
if (progressFill) progressFill.style.width = '0%';
|
||||
if (progressCurrent) progressCurrent.textContent = '0';
|
||||
if (progressTotal) progressTotal.textContent = '0';
|
||||
if (currentAddress) currentAddress.textContent = '';
|
||||
|
||||
// Destroy existing map if it exists
|
||||
if (resultsMap) {
|
||||
try {
|
||||
resultsMap.remove();
|
||||
resultsMap = null;
|
||||
} catch (e) {
|
||||
console.warn('Error removing map:', e);
|
||||
}
|
||||
}
|
||||
|
||||
// Reset form
|
||||
clearUpload();
|
||||
}
|
||||
@@ -2,6 +2,7 @@ const express = require('express');
|
||||
const router = express.Router();
|
||||
const settingsController = require('../controllers/settingsController');
|
||||
const dashboardRoutes = require('./dashboard');
|
||||
const dataConvertRoutes = require('./dataConvert');
|
||||
|
||||
// Debug endpoint to check configuration
|
||||
router.get('/config-debug', (req, res) => {
|
||||
@@ -34,4 +35,7 @@ router.post('/walk-sheet-config', settingsController.updateWalkSheetConfig);
|
||||
// Dashboard routes
|
||||
router.use('/dashboard', dashboardRoutes);
|
||||
|
||||
// Data convert routes
|
||||
router.use('/data-convert', dataConvertRoutes);
|
||||
|
||||
module.exports = router;
|
||||
27
map/app/routes/dataConvert.js
Normal file
27
map/app/routes/dataConvert.js
Normal file
@@ -0,0 +1,27 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const multer = require('multer');
|
||||
const dataConvertController = require('../controllers/dataConvertController');
|
||||
|
||||
// Configure multer for CSV upload
|
||||
const upload = multer({
|
||||
storage: multer.memoryStorage(),
|
||||
fileFilter: (req, file, cb) => {
|
||||
if (file.mimetype === 'text/csv' || file.originalname.endsWith('.csv')) {
|
||||
cb(null, true);
|
||||
} else {
|
||||
cb(new Error('Only CSV files are allowed'));
|
||||
}
|
||||
},
|
||||
limits: {
|
||||
fileSize: 10 * 1024 * 1024 // 10MB limit
|
||||
}
|
||||
});
|
||||
|
||||
// Process CSV file
|
||||
router.post('/process-csv', upload.single('csvFile'), dataConvertController.processCSV);
|
||||
|
||||
// Save geocoded data
|
||||
router.post('/save-geocoded', dataConvertController.saveGeocodedData);
|
||||
|
||||
module.exports = router;
|
||||
@@ -112,8 +112,8 @@ async function forwardGeocode(address) {
|
||||
}
|
||||
|
||||
try {
|
||||
// Add delay to respect rate limits
|
||||
await new Promise(resolve => setTimeout(resolve, 1000));
|
||||
// Add delay to respect rate limits - increase delay for batch processing
|
||||
await new Promise(resolve => setTimeout(resolve, 1500));
|
||||
|
||||
logger.info(`Forward geocoding: ${address}`);
|
||||
|
||||
@@ -126,9 +126,9 @@ async function forwardGeocode(address) {
|
||||
'accept-language': 'en'
|
||||
},
|
||||
headers: {
|
||||
'User-Agent': 'NocoDB Map Viewer 1.0 (https://github.com/yourusername/nocodb-map-viewer)'
|
||||
'User-Agent': 'NocoDB Map Viewer 1.0 (contact@example.com)'
|
||||
},
|
||||
timeout: 10000
|
||||
timeout: 15000 // Increase timeout to 15 seconds
|
||||
});
|
||||
|
||||
if (!response.data || response.data.length === 0) {
|
||||
@@ -151,10 +151,16 @@ async function forwardGeocode(address) {
|
||||
|
||||
if (error.response?.status === 429) {
|
||||
throw new Error('Rate limit exceeded. Please try again later.');
|
||||
} else if (error.response?.status === 403) {
|
||||
throw new Error('Access denied by geocoding service');
|
||||
} else if (error.response?.status === 500) {
|
||||
throw new Error('Geocoding service internal error');
|
||||
} else if (error.code === 'ECONNABORTED') {
|
||||
throw new Error('Geocoding service timeout');
|
||||
throw new Error('Geocoding request timeout');
|
||||
} else if (error.code === 'ENOTFOUND' || error.code === 'ECONNREFUSED') {
|
||||
throw new Error('Cannot connect to geocoding service');
|
||||
} else {
|
||||
throw new Error('Geocoding service unavailable');
|
||||
throw new Error(`Geocoding failed: ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user