updates and fixes

This commit is contained in:
2025-08-25 10:32:42 -06:00
parent 785c7471c0
commit ee1f6fbf5c
16 changed files with 1075 additions and 5261 deletions

View File

@@ -1,148 +0,0 @@
#!/bin/bash
#
# Enhanced Conversations Deployment Script
# Deploys all enhanced conversation components
#
set -e
echo "🚀 Deploying Enhanced Conversations System..."
# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m' # No Color
# Check if we're in the project root
if [ ! -f "src/app.py" ]; then
echo -e "${RED}❌ Please run this script from the project root directory${NC}"
exit 1
fi
echo -e "${BLUE}📋 Step 1: Running database migration...${NC}"
python3 scripts/migrate_conversations_db.py
if [ $? -eq 0 ]; then
echo -e "${GREEN}✅ Database migration completed${NC}"
else
echo -e "${RED}❌ Database migration failed${NC}"
exit 1
fi
echo -e "${BLUE}📋 Step 2: Integrating with main application...${NC}"
python3 scripts/integrate_enhanced_conversations.py
if [ $? -eq 0 ]; then
echo -e "${GREEN}✅ Application integration completed${NC}"
else
echo -e "${RED}❌ Application integration failed${NC}"
exit 1
fi
echo -e "${BLUE}📋 Step 3: Installing Python dependencies...${NC}"
cd src
# Try different installation methods
if command -v pip3 >/dev/null 2>&1; then
echo "Installing with pip3..."
pip3 install -r requirements.txt --break-system-packages --user
if [ $? -ne 0 ]; then
echo -e "${YELLOW}⚠️ Pip installation failed, trying with virtual environment...${NC}"
# Try with virtual environment
if ! command -v python3-venv >/dev/null 2>&1; then
echo "Installing python3-venv..."
sudo apt-get update && sudo apt-get install -y python3-venv
fi
if [ ! -d "../venv" ]; then
echo "Creating virtual environment..."
python3 -m venv ../venv
fi
echo "Installing dependencies in virtual environment..."
../venv/bin/pip install -r requirements.txt
fi
else
echo -e "${YELLOW}⚠️ Pip not found, dependencies may need manual installation${NC}"
fi
echo -e "${BLUE}📋 Step 3: Skipping Python dependencies (using Docker Compose)...${NC}"
echo -e "${GREEN}✅ Dependencies handled by Docker${NC}"
echo -e "${BLUE}📋 Step 4: Updating Android Termux API server...${NC}"
PHONE_IP=${PHONE_IP:-"10.0.0.193"}
PHONE_PORT=${PHONE_PORT:-"8022"}
PHONE_USER=${PHONE_USER:-"android-dev"}
# Check if Android device is reachable
if ping -c 1 -W 1 $PHONE_IP > /dev/null 2>&1; then
echo -e "${GREEN}📱 Android device is reachable at $PHONE_IP${NC}"
# Deploy updated Termux API server
echo "Uploading enhanced Termux API server..."
scp -P $PHONE_PORT android/termux-sms-api-server.py $PHONE_USER@$PHONE_IP:~/projects/sms-campaign-manager/
if [ $? -eq 0 ]; then
echo -e "${GREEN}✅ Termux API server updated${NC}"
# Restart the server
echo "Restarting Termux API server..."
ssh -p $PHONE_PORT $PHONE_USER "pkill -f termux-sms-api-server.py; cd ~/projects/sms-campaign-manager && python termux-sms-api-server.py > /dev/null 2>&1 &"
sleep 2
# Test the server
if curl -s http://$PHONE_IP:5001/health > /dev/null; then
echo -e "${GREEN}✅ Termux API server is running${NC}"
else
echo -e "${YELLOW}⚠️ Termux API server may need manual restart${NC}"
fi
else
echo -e "${YELLOW}⚠️ Could not update Termux API server - manual update required${NC}"
fi
else
echo -e "${YELLOW}⚠️ Android device not reachable - manual Termux server update required${NC}"
echo "Manually copy android/termux-sms-api-server.py to your Android device"
fi
echo -e "${BLUE}📋 Step 5: Testing the system...${NC}"
# Test database
echo "Testing database schema..."
if sqlite3 data/campaign.db ".schema conversations" | grep -q "is_starred"; then
echo -e "${GREEN}✅ Database schema is correct${NC}"
else
echo -e "${RED}❌ Database schema appears incorrect${NC}"
fi
# Test API endpoints
echo "Testing local API endpoints..."
if curl -s http://localhost:5000/health > /dev/null 2>&1; then
echo -e "${GREEN}✅ Main application is responding${NC}"
else
echo -e "${YELLOW}⚠️ Main application is not running${NC}"
fi
echo -e "${GREEN}🎉 Enhanced Conversations Deployment Complete!${NC}"
echo ""
echo -e "${BLUE}📋 What's New:${NC}"
echo "• WhatsApp-style conversation interface"
echo "• Real-time message updates via WebSocket"
echo "• Bidirectional SMS sync with Android device"
echo "• Message status tracking (pending, sent, delivered, failed)"
echo "• Contact name resolution from phone"
echo "• Conversation starring and importance marking"
echo "• Scrollable message history with pagination"
echo "• Manual message sending from conversation view"
echo ""
echo -e "${BLUE}🚀 Next Steps:${NC}"
echo "1. Start the application: docker-compose up -d OR python src/app.py"
echo "2. Open http://localhost:5000 in your browser"
echo "3. Go to the Conversations tab"
echo "4. Test sending messages and real-time sync"
echo ""
echo -e "${BLUE}🔧 If you encounter issues:${NC}"
echo "• Check logs: docker-compose logs -f"
echo "• Verify Android device connectivity"
echo "• Test Termux API: curl http://$PHONE_IP:5001/health"
echo "• Check WebSocket connection in browser dev tools"

View File

@@ -1,181 +0,0 @@
#!/usr/bin/env python3
"""
Integration Script - Update main app.py for enhanced conversations
"""
import os
def update_main_app():
"""Add the necessary imports and initialization for enhanced conversations"""
app_py_path = './src/app.py'
# Read current app.py
with open(app_py_path, 'r') as f:
content = f.read()
# Add imports at the beginning
imports_to_add = """
# Enhanced conversations imports
from services.termux_sync_service import TermuxSyncService
from services.websocket_service import WebSocketService
from routes.conversations_enhanced import conversations_enhanced_bp, set_services
import threading
import asyncio
import logging
"""
# Add initialization code before app.run
init_code = """
# Initialize enhanced conversation services
try:
# Get phone IP from environment
phone_ip = os.getenv('PHONE_IP', '10.0.0.193')
termux_port = os.getenv('TERMUX_API_PORT', '5001')
termux_api_url = f"http://{phone_ip}:{termux_port}"
# Initialize sync service
sync_service = TermuxSyncService(termux_api_url)
# Initialize WebSocket service
websocket_service = WebSocketService(app, sync_service)
sync_service.set_websocket_service(websocket_service)
# Set service references for enhanced routes
set_services(sync_service, websocket_service)
# Register enhanced conversations blueprint
app.register_blueprint(conversations_enhanced_bp)
# Start background sync service
def run_sync_service():
try:
loop = asyncio.new_event_loop()
asyncio.set_event_loop(loop)
loop.run_until_complete(sync_service.start_sync_loop())
except Exception as e:
logging.error(f"Sync service error: {e}")
sync_thread = threading.Thread(target=run_sync_service, daemon=True)
sync_thread.start()
logging.info("🚀 Enhanced conversation services initialized")
except Exception as e:
logging.error(f"Failed to initialize enhanced conversation services: {e}")
"""
# Check if already integrated
if 'termux_sync_service' in content:
print("✅ App already appears to be integrated with enhanced conversations")
return
# Add imports after existing imports
import_insertion_point = content.find('from models.conversation import Conversation')
if import_insertion_point != -1:
content = content[:import_insertion_point] + imports_to_add + '\n' + content[import_insertion_point:]
# Add initialization before app.run
app_run_point = content.find('app.run(')
if app_run_point != -1:
content = content[:app_run_point] + init_code + '\n' + content[app_run_point:]
# Write updated content
with open(app_py_path, 'w') as f:
f.write(content)
print("✅ Successfully updated app.py with enhanced conversation support")
def update_dashboard_template():
"""Update dashboard template to include enhanced conversations"""
template_path = './src/templates/dashboard.html'
enhanced_template_path = './src/templates/conversations_enhanced.html'
# Read the enhanced template
with open(enhanced_template_path, 'r') as f:
enhanced_content = f.read()
# Read the main dashboard
with open(template_path, 'r') as f:
dashboard_content = f.read()
# Find and replace the conversations tab
start_marker = '<!-- Conversations Tab -->'
end_marker = '<!-- End Conversations Tab -->'
if start_marker not in dashboard_content:
# Look for the existing conversations tab
start_marker = '<div x-show="activeTab === \'conversations\'"'
end_marker = '</div>\n </div>'
start_pos = dashboard_content.find(start_marker)
if start_pos == -1:
print("❌ Could not find conversations tab in dashboard.html")
return
# Find the end of the conversations div
div_count = 0
end_pos = start_pos
in_div = False
for i, char in enumerate(dashboard_content[start_pos:], start_pos):
if char == '<':
if dashboard_content[i:i+4] == '<div':
div_count += 1
in_div = True
elif dashboard_content[i:i+6] == '</div>':
div_count -= 1
if div_count == 0 and in_div:
end_pos = i + 6
break
else:
start_pos = dashboard_content.find(start_marker)
end_pos = dashboard_content.find(end_marker, start_pos) + len(end_marker)
# Replace with enhanced template
if start_pos != -1 and end_pos != -1:
new_content = (dashboard_content[:start_pos] +
'<!-- Enhanced Conversations Tab -->\n' +
enhanced_content +
'\n<!-- End Enhanced Conversations Tab -->\n' +
dashboard_content[end_pos:])
# Add enhanced JavaScript import
if 'conversations_enhanced.js' not in new_content:
js_insertion_point = new_content.find('<script src="/static/js/conversations.js"></script>')
if js_insertion_point != -1:
new_content = (new_content[:js_insertion_point] +
'<script src="/static/js/conversations_enhanced.js"></script>\n' +
new_content[js_insertion_point:])
# Add Socket.IO CDN
if 'socket.io' not in new_content:
head_end = new_content.find('</head>')
if head_end != -1:
new_content = (new_content[:head_end] +
' <script src="https://cdn.socket.io/4.7.2/socket.io.min.js"></script>\n' +
new_content[head_end:])
with open(template_path, 'w') as f:
f.write(new_content)
print("✅ Successfully updated dashboard.html with enhanced conversations")
else:
print("❌ Could not locate conversations section in dashboard.html")
if __name__ == "__main__":
print("🔄 Integrating enhanced conversations...")
# Update main application
update_main_app()
# Update dashboard template
update_dashboard_template()
print("✅ Integration complete!")
print("\n📝 Next steps:")
print("1. Run database migration: python scripts/migrate_conversations_db.py")
print("2. Update Android Termux API server")
print("3. Restart the application")
print("4. Test the enhanced conversations interface")

View File

@@ -1,78 +0,0 @@
#!/usr/bin/env python3
"""
Database Migration Script - Enhance conversations schema for WhatsApp-style features
"""
import sqlite3
import logging
from datetime import datetime
logger = logging.getLogger(__name__)
def migrate_conversations_schema(db_path='./data/campaign.db'):
"""Add new fields to support enhanced conversations"""
conn = sqlite3.connect(db_path)
cursor = conn.cursor()
migrations = [
# Add new fields to conversations table
("ALTER TABLE conversations ADD COLUMN contact_name TEXT DEFAULT ''", "contact_name field"),
("ALTER TABLE conversations ADD COLUMN is_starred BOOLEAN DEFAULT FALSE", "is_starred field"),
("ALTER TABLE conversations ADD COLUMN last_sync_timestamp INTEGER DEFAULT 0", "last_sync_timestamp field"),
("ALTER TABLE conversations ADD COLUMN total_message_count INTEGER DEFAULT 0", "total_message_count field"),
# Add new fields to messages table for status tracking
("ALTER TABLE messages ADD COLUMN status TEXT DEFAULT 'pending' CHECK(status IN ('pending', 'sent', 'delivered', 'failed'))", "message status field"),
("ALTER TABLE messages ADD COLUMN external_message_id TEXT", "external_message_id field"),
("ALTER TABLE messages ADD COLUMN sync_status TEXT DEFAULT 'local' CHECK(sync_status IN ('local', 'synced', 'pending_sync'))", "sync_status field"),
("ALTER TABLE messages ADD COLUMN direction TEXT DEFAULT 'outbound' CHECK(direction IN ('inbound', 'outbound'))", "direction field"),
("ALTER TABLE messages ADD COLUMN timestamp INTEGER", "timestamp field"),
# Create indexes for performance
("CREATE INDEX IF NOT EXISTS idx_messages_timestamp ON messages(timestamp)", "timestamp index"),
("CREATE INDEX IF NOT EXISTS idx_conversations_starred ON conversations(is_starred)", "starred conversations index"),
("CREATE INDEX IF NOT EXISTS idx_messages_status ON messages(status)", "message status index"),
("CREATE INDEX IF NOT EXISTS idx_messages_direction ON messages(direction)", "message direction index"),
("CREATE INDEX IF NOT EXISTS idx_messages_sync_status ON messages(sync_status)", "sync status index"),
]
for migration_sql, description in migrations:
try:
cursor.execute(migration_sql)
logger.info(f"✅ Applied: {description}")
except sqlite3.OperationalError as e:
if "duplicate column name" in str(e) or "already exists" in str(e):
logger.info(f"⏭️ Skipped: {description} (already exists)")
else:
logger.error(f"❌ Failed: {description} - {e}")
raise
# Update existing messages to have proper timestamp and direction
cursor.execute("""
UPDATE messages
SET timestamp = CAST(strftime('%s', sent_at) AS INTEGER)
WHERE timestamp IS NULL AND sent_at IS NOT NULL
""")
cursor.execute("""
UPDATE messages
SET timestamp = CAST(strftime('%s', 'now') AS INTEGER)
WHERE timestamp IS NULL
""")
cursor.execute("""
UPDATE messages
SET direction = CASE
WHEN response_text IS NOT NULL AND response_text != '' THEN 'inbound'
ELSE 'outbound'
END
WHERE direction IS NULL
""")
conn.commit()
conn.close()
logger.info("🚀 Database migration completed successfully!")
if __name__ == "__main__":
logging.basicConfig(level=logging.INFO)
migrate_conversations_schema()

View File

@@ -1,278 +0,0 @@
#!/usr/bin/env python3
"""
Enhanced Conversations Test Suite
Test all components of the enhanced conversation system
"""
import requests
import sqlite3
import json
import time
import logging
from typing import Dict, List, Optional
# Configure logging
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
class EnhancedConversationTester:
"""Test suite for enhanced conversation features"""
def __init__(self, base_url='http://localhost:5000', phone_ip='10.0.0.193'):
self.base_url = base_url.rstrip('/')
self.phone_ip = phone_ip
self.phone_api_url = f'http://{phone_ip}:5001'
self.db_path = './data/campaign.db'
self.test_phone = '5551234567' # Test phone number
def run_all_tests(self):
"""Run comprehensive test suite"""
logger.info("🧪 Starting Enhanced Conversations Test Suite")
tests = [
("Database Schema", self.test_database_schema),
("API Endpoints", self.test_api_endpoints),
("WebSocket Connection", self.test_websocket_connection),
("Android API Connection", self.test_android_api_connection),
("Message Flow", self.test_message_flow),
("Conversation Management", self.test_conversation_management),
("Real-time Updates", self.test_realtime_updates)
]
passed = 0
failed = 0
for test_name, test_func in tests:
try:
logger.info(f"🔍 Testing: {test_name}")
test_func()
logger.info(f"{test_name}: PASSED")
passed += 1
except Exception as e:
logger.error(f"{test_name}: FAILED - {e}")
failed += 1
# Summary
total = passed + failed
success_rate = (passed / total * 100) if total > 0 else 0
logger.info(f"\n📊 Test Results: {passed}/{total} passed ({success_rate:.1f}%)")
if failed > 0:
logger.warning(f"⚠️ {failed} tests failed - check logs for details")
else:
logger.info("🎉 All tests passed!")
return failed == 0
def test_database_schema(self):
"""Test database schema has required fields"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# Check conversations table
cursor.execute("PRAGMA table_info(conversations)")
columns = [row[1] for row in cursor.fetchall()]
required_columns = ['is_starred', 'contact_name', 'last_sync_timestamp']
for col in required_columns:
if col not in columns:
raise Exception(f"Missing column '{col}' in conversations table")
# Check messages table
cursor.execute("PRAGMA table_info(messages)")
columns = [row[1] for row in cursor.fetchall()]
required_columns = ['status', 'direction', 'timestamp', 'sync_status']
for col in required_columns:
if col not in columns:
raise Exception(f"Missing column '{col}' in messages table")
conn.close()
def test_api_endpoints(self):
"""Test enhanced API endpoints"""
endpoints = [
'/api/conversations/enhanced',
'/api/conversations/stats',
]
for endpoint in endpoints:
response = requests.get(f"{self.base_url}{endpoint}")
if response.status_code != 200:
raise Exception(f"Endpoint {endpoint} returned {response.status_code}")
data = response.json()
if not data.get('success'):
raise Exception(f"Endpoint {endpoint} returned error: {data.get('error')}")
def test_websocket_connection(self):
"""Test WebSocket server is available"""
try:
# Test if Socket.IO endpoint responds
response = requests.get(f"{self.base_url}/socket.io/?EIO=4&transport=polling")
if response.status_code not in [200, 400]: # 400 is expected for wrong transport
raise Exception(f"WebSocket server not responding: {response.status_code}")
except requests.RequestException as e:
raise Exception(f"WebSocket server connection failed: {e}")
def test_android_api_connection(self):
"""Test Android Termux API connection"""
try:
response = requests.get(f"{self.phone_api_url}/health", timeout=5)
if response.status_code != 200:
raise Exception(f"Android API health check failed: {response.status_code}")
data = response.json()
if not data.get('success'):
raise Exception(f"Android API health check error: {data.get('error')}")
# Test enhanced endpoints
response = requests.get(f"{self.phone_api_url}/api/sms/history?limit=1", timeout=5)
if response.status_code != 200:
logger.warning("SMS history endpoint may not be available")
except requests.RequestException as e:
raise Exception(f"Android API connection failed: {e}")
def test_message_flow(self):
"""Test message creation and management"""
# Create a test conversation
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
conversation_id = f"test_conv_{int(time.time())}"
# Insert test conversation
cursor.execute("""
INSERT INTO conversations (id, phone, contact_name, status, created_at)
VALUES (?, ?, 'Test Contact', 'active', datetime('now'))
""", (conversation_id, self.test_phone))
# Insert test message
cursor.execute("""
INSERT INTO messages (conversation_id, phone, message, direction, status, timestamp)
VALUES (?, ?, 'Test message', 'outbound', 'pending', ?)
""", (conversation_id, self.test_phone, int(time.time())))
conn.commit()
# Test API retrieval
response = requests.get(f"{self.base_url}/api/conversations/{conversation_id}/messages")
if response.status_code != 200:
conn.close()
raise Exception(f"Failed to retrieve messages: {response.status_code}")
data = response.json()
if not data.get('success') or not data.get('messages'):
conn.close()
raise Exception("No messages returned from API")
# Clean up
cursor.execute("DELETE FROM conversations WHERE id = ?", (conversation_id,))
cursor.execute("DELETE FROM messages WHERE conversation_id = ?", (conversation_id,))
conn.commit()
conn.close()
def test_conversation_management(self):
"""Test conversation CRUD operations"""
# Create test conversation
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
conversation_id = f"test_conv_mgmt_{int(time.time())}"
cursor.execute("""
INSERT INTO conversations (id, phone, contact_name, status, is_starred, created_at)
VALUES (?, ?, 'Test Management', 'active', 0, datetime('now'))
""", (conversation_id, self.test_phone))
conn.commit()
try:
# Test starring
response = requests.put(f"{self.base_url}/api/conversations/{conversation_id}/star")
if response.status_code != 200:
raise Exception(f"Failed to toggle star: {response.status_code}")
data = response.json()
if not data.get('success'):
raise Exception(f"Star toggle error: {data.get('error')}")
# Verify starring worked
cursor.execute("SELECT is_starred FROM conversations WHERE id = ?", (conversation_id,))
row = cursor.fetchone()
if not row or not row[0]:
raise Exception("Conversation was not starred")
# Test mark as read
response = requests.put(f"{self.base_url}/api/conversations/{conversation_id}/mark-read")
if response.status_code != 200:
raise Exception(f"Failed to mark as read: {response.status_code}")
finally:
# Clean up
cursor.execute("DELETE FROM conversations WHERE id = ?", (conversation_id,))
cursor.execute("DELETE FROM messages WHERE conversation_id = ?", (conversation_id,))
conn.commit()
conn.close()
def test_realtime_updates(self):
"""Test real-time update mechanisms"""
# This is a basic test - full WebSocket testing would require a client
try:
# Test sync endpoints
response = requests.post(f"{self.base_url}/api/conversations/sync-all")
if response.status_code not in [200, 503]: # 503 if sync service unavailable
raise Exception(f"Sync all endpoint failed: {response.status_code}")
if response.status_code == 200:
data = response.json()
if not data.get('success'):
raise Exception(f"Sync all error: {data.get('error')}")
except requests.RequestException as e:
raise Exception(f"Real-time update test failed: {e}")
def test_data_integrity(self):
"""Test data integrity and constraints"""
conn = sqlite3.connect(self.db_path)
cursor = conn.cursor()
# Test that we can't create invalid status values
try:
cursor.execute("""
INSERT INTO messages (phone, message, status, direction)
VALUES (?, 'test', 'invalid_status', 'outbound')
""", (self.test_phone,))
conn.commit()
raise Exception("Database allowed invalid status value")
except sqlite3.IntegrityError:
# This is expected
pass
conn.close()
def main():
"""Run the test suite"""
import argparse
parser = argparse.ArgumentParser(description='Test Enhanced Conversations System')
parser.add_argument('--base-url', default='http://localhost:5000',
help='Base URL for the main application')
parser.add_argument('--phone-ip', default='10.0.0.193',
help='IP address of Android device')
parser.add_argument('--verbose', action='store_true',
help='Enable verbose logging')
args = parser.parse_args()
if args.verbose:
logging.getLogger().setLevel(logging.DEBUG)
tester = EnhancedConversationTester(args.base_url, args.phone_ip)
success = tester.run_all_tests()
return 0 if success else 1
if __name__ == "__main__":
exit(main())