admin/changemaker.lite
1
0
Fork 0
Code Issues Packages Projects Releases 35 Wiki Activity

updates to the build system

Browse Source
This commit is contained in:
admin
2025-09-10 17:35:48 -06:00
parent 56b1600c37
commit 609f89ec0c
2 changed files with 453 additions and 89 deletions
Download Patch File Download Diff File Expand all files Collapse all files

115
map/build-nocodb.sh
View File

@@ -60,6 +60,7 @@ USAGE:
OPTIONS:
--migrate-data Skip interactive prompt and enable data migration mode
--help Show this help message
--test-export Test export functionality (for debugging)
DESCRIPTION:
This script creates a new NocoDB base with the required tables for the Map Viewer application.
@@ -92,6 +93,12 @@ parse_arguments() {
MIGRATE_DATA=true
shift
;;
--test-export)
print_status "Test mode - export functionality verification"
print_status "This would test the pagination logic with your current setup"
print_warning "Test mode not yet implemented - use normal migration to test"
exit 0
;;
--help)
show_usage
exit 0
@@ -336,20 +343,95 @@ export_table_data() {
local base_id=$1
local table_id=$2
local table_name=$3
local limit=${4:-1000} # Default limit of 1000 records
print_status "Exporting data from table: $table_name (ID: $table_id)"
local response
response=$(make_api_call "GET" "/tables/$table_id/records?limit=$limit" "" "Exporting data from $table_name" "v2")
# First, get total count of records using a minimal request
local count_response
count_response=$(make_api_call "GET" "/tables/$table_id/records?limit=1" "" "Getting record count for $table_name" "v2")
if [[ $? -eq 0 && -n "$response" ]]; then
echo "$response"
return 0
else
print_error "Failed to export data from table: $table_name"
if [[ $? -ne 0 ]]; then
print_error "Failed to get record count for table: $table_name"
return 1
fi
# Extract total count from pageInfo
local total_count
total_count=$(echo "$count_response" | jq -r '.pageInfo.totalRows // 0' 2>/dev/null)
if [[ -z "$total_count" || "$total_count" == "null" || "$total_count" -eq 0 ]]; then
print_warning "No records found in table: $table_name"
echo '{"list":[],"pageInfo":{"totalRows":0}}'
return 0
fi
print_status "Found $total_count records in table: $table_name"
# If we have a small number of records, get them all at once
if [[ "$total_count" -le 1000 ]]; then
local response
response=$(make_api_call "GET" "/tables/$table_id/records?limit=$total_count" "" "Exporting all $total_count records from $table_name" "v2")
if [[ $? -eq 0 && -n "$response" ]]; then
echo "$response"
return 0
else
print_error "Failed to export data from table: $table_name"
return 1
fi
else
# For larger datasets, paginate through all records
print_status "Large dataset detected. Paginating through all $total_count records..."
local all_records="[]"
local offset=0
local limit=1000
local batch_num=1
while [[ $offset -lt $total_count ]]; do
local remaining=$((total_count - offset))
local current_limit=$limit
if [[ $remaining -lt $limit ]]; then
current_limit=$remaining
fi
print_status "Fetching batch $batch_num: records $((offset + 1)) to $((offset + current_limit)) of $total_count"
local batch_response
batch_response=$(make_api_call "GET" "/tables/$table_id/records?limit=$current_limit&offset=$offset" "" "Fetching batch $batch_num" "v2")
if [[ $? -ne 0 ]]; then
print_error "Failed to fetch batch $batch_num from table: $table_name"
return 1
fi
# Extract records from this batch and merge with all_records
local batch_records
batch_records=$(echo "$batch_response" | jq -r '.list' 2>/dev/null)
if [[ -n "$batch_records" && "$batch_records" != "null" && "$batch_records" != "[]" ]]; then
all_records=$(echo "$all_records $batch_records" | jq -s 'add' 2>/dev/null)
fi
offset=$((offset + current_limit))
batch_num=$((batch_num + 1))
# Small delay to avoid overwhelming the API, longer delay for larger datasets
if [[ $total_count -gt 5000 ]]; then
sleep 1
else
sleep 0.5
fi
done
# Return the complete dataset in the expected format
local final_response
final_response=$(jq -n --argjson records "$all_records" --argjson total "$total_count" '{"list": $records, "pageInfo": {"totalRows": $total}}')
print_success "Successfully exported all $total_count records from table: $table_name"
echo "$final_response"
return 0
fi
}
# Function to import data into a table
@@ -381,7 +463,7 @@ import_table_data() {
# Count total records first
local total_records
total_records=$(echo "$records_array" | jq 'length' 2>/dev/null)
print_status "Found $total_records records to import"
print_status "Found $total_records records to import into $table_name"
local import_count=0
local success_count=0
@@ -390,6 +472,15 @@ import_table_data() {
local temp_file="/tmp/nocodb_import_$$"
echo "0" > "$temp_file"
# Add progress reporting for large datasets
local progress_interval=50
if [[ $total_records -gt 200 ]]; then
progress_interval=100
fi
if [[ $total_records -gt 1000 ]]; then
progress_interval=250
fi
# Parse records and import them one by one (to handle potential ID conflicts)
echo "$records_array" | jq -c '.[]' 2>/dev/null | while read -r record; do
import_count=$((import_count + 1))
@@ -430,7 +521,11 @@ import_table_data() {
success_count=$(cat "$temp_file")
success_count=$((success_count + 1))
echo "$success_count" > "$temp_file"
print_status "✓ Imported record $import_count/$total_records"
# Show progress at intervals to avoid too much output
if [[ $((import_count % progress_interval)) -eq 0 ]] || [[ $import_count -eq $total_records ]]; then
print_status "✓ Imported $import_count/$total_records records ($(($success_count * 100 / import_count))% success rate)"
fi
else
local response_body="${response%???}"
print_warning "✗ Failed to import record $import_count/$total_records: $response_body"