Monitoring & Analytics

Comprehensive monitoring and analytics for Earna AI Console. Track chat performance, GPT-4o API usage, alternative model metrics, Supabase database health, and user engagement.

Overview

Earna AI Console monitoring provides visibility into:

Chat Analytics: Message volume, response times, and user satisfaction
GPT-4o Performance: API latency, token usage, and cost tracking
Alternative Models: Claude, Gemini, and other model performance
Supabase Health: Database connections, query performance, real-time subscriptions
Avatar & Voice: HeyGen sessions and GPT-4o Realtime voice metrics
User Engagement: Daily limits, subscription tiers, and feature usage

Analytics Dashboard

Real-time Metrics

Monitor critical platform metrics in real-time:


// lib/monitoring/platform-metrics.ts
export interface PlatformMetrics {
  chat: {
    activeSessions: number;
    messagesProcessed: number;
    avgResponseTime: number;  // ms
    timeToFirstToken: number; // ms for streaming
    satisfaction: number;      // 0-5 rating
    successRate: number;       // 0-100
  };
  gpt4o: {
    requestsPerHour: number;
    avgLatency: number;        // ms
    tokenUsage: number;
    costPerHour: number;       // USD
    streamingActive: number;   // active streams
    errorRate: number;         // percentage
  };
  alternativeModels: {
    claude: ModelMetrics;
    gemini: ModelMetrics;
    mistral: ModelMetrics;
    activeModel: string;       // currently selected
    fallbackCount: number;     // fallback to GPT-4o
  };
  supabase: {
    databaseHealth: 'healthy' | 'degraded' | 'down';
    activeConnections: number;
    queryLatency: number;      // ms
    realtimeSubscriptions: number;
    storageUsage: number;      // MB
  };
  features: {
    avatarSessions: number;    // HeyGen active sessions
    voiceSessions: number;     // GPT-4o Realtime voice
    fileUploads: number;
    dailyLimits: {
      free: { used: number; limit: number };
      pro: { used: number; limit: number };
    };
  };
  system: {
    apiLatency: number;        // ms
    errorRate: number;         // percentage
    uptime: number;            // percentage
    activeUsers: number;
    edgeFunctionCalls: number;
    cacheHitRate: number;      // percentage
  };
}
 
export interface ModelMetrics {
  requests: number;
  avgLatency: number;
  tokenUsage: number;
  cost: number;
  errorRate: number;
  availability: boolean;
}
 
export class PlatformAnalytics {
  private static instance: PlatformAnalytics;
  private metrics: PlatformMetrics;
  private subscribers: ((metrics: PlatformMetrics) => void)[] = [];
 
  static getInstance(): PlatformAnalytics {
    if (!PlatformAnalytics.instance) {
      PlatformAnalytics.instance = new PlatformAnalytics();
    }
    return PlatformAnalytics.instance;
  }
 
  async collectMetrics(): Promise<PlatformMetrics> {
    this.metrics = {
      chat: await this.collectChatMetrics(),
      gpt4o: await this.collectGPT4oMetrics(),
      alternativeModels: await this.collectAlternativeModelMetrics(),
      supabase: await this.collectSupabaseMetrics(),
      features: await this.collectFeatureMetrics(),
      system: await this.collectSystemMetrics()
    };
 
    // Notify subscribers
    this.subscribers.forEach(callback => callback(this.metrics));
    
    return this.metrics;
  }
 
  subscribe(callback: (metrics: PlatformMetrics) => void): () => void {
    this.subscribers.push(callback);
    return () => {
      this.subscribers = this.subscribers.filter(cb => cb !== callback);
    };
  }
 
  private async collectChatMetrics() {
    const timeWindow = new Date(Date.now() - 60 * 60 * 1000); // Last hour
    
    const [sessions, messages, responseTime, ttft, satisfaction] = await Promise.all([
      this.getActiveChatSessions(),
      this.getMessagesProcessed(timeWindow),
      this.getAverageResponseTime(timeWindow),
      this.getTimeToFirstToken(timeWindow),
      this.getAverageSatisfaction(timeWindow)
    ]);
 
    return {
      activeSessions: sessions,
      messagesProcessed: messages,
      avgResponseTime: responseTime,
      timeToFirstToken: ttft,
      satisfaction,
      successRate: this.calculateChatSuccessRate(messages)
    };
  }
 
  private async collectGPT4oMetrics() {
    const hourAgo = new Date(Date.now() - 60 * 60 * 1000);
    
    return {
      requestsPerHour: await this.getGPT4oRequests(hourAgo),
      avgLatency: await this.getGPT4oLatency(hourAgo),
      tokenUsage: await this.getTokenUsage(hourAgo),
      costPerHour: await this.calculateGPT4oCost(hourAgo),
      streamingActive: await this.getActiveStreams(),
      errorRate: await this.getGPT4oErrorRate(hourAgo)
    };
  }
 
  private async collectSupabaseMetrics() {
    const [health, connections, latency, subscriptions, storage] = await Promise.all([
      this.checkSupabaseHealth(),
      this.getActiveConnections(),
      this.getQueryLatency(),
      this.getRealtimeSubscriptions(),
      this.getStorageUsage()
    ]);
    
    return {
      databaseHealth: health,
      activeConnections: connections,
      queryLatency: latency,
      realtimeSubscriptions: subscriptions,
      storageUsage: storage
    };
  }
}

Monitoring Dashboard Component


// components/monitoring/Dashboard.tsx
'use client';
 
import { useState, useEffect } from 'react';
import { Card, CardContent, CardHeader, CardTitle } from '@/components/ui/card';
import { Badge } from '@/components/ui/badge';
import { Progress } from '@/components/ui/progress';
import { PlatformAnalytics, PlatformMetrics } from '@/lib/monitoring/platform-metrics';
 
export function MonitoringDashboard() {
  const [metrics, setMetrics] = useState<PlatformMetrics | null>(null);
  const [loading, setLoading] = useState(true);
  
  const analytics = PlatformAnalytics.getInstance();
 
  useEffect(() => {
    const loadMetrics = async () => {
      const currentMetrics = await analytics.collectMetrics();
      setMetrics(currentMetrics);
      setLoading(false);
    };
 
    loadMetrics();
 
    // Subscribe to real-time updates
    const unsubscribe = analytics.subscribe(setMetrics);
 
    // Refresh every 30 seconds
    const interval = setInterval(loadMetrics, 30000);
 
    return () => {
      unsubscribe();
      clearInterval(interval);
    };
  }, []);
 
  if (loading) return <div>Loading metrics...</div>;
  if (!metrics) return <div>Failed to load metrics</div>;
 
  return (
    <div className="grid grid-cols-1 md:grid-cols-2 lg:grid-cols-4 gap-6">
      {/* Chat Metrics */}
      <Card>
        <CardHeader>
          <CardTitle>Chat Activity</CardTitle>
        </CardHeader>
        <CardContent>
          <div className="text-3xl font-bold">{metrics.chat.activeSessions}</div>
          <div className="text-sm text-muted-foreground">
            {metrics.chat.messagesProcessed} messages today
          </div>
          <div className="mt-2 space-y-1">
            <Badge variant={metrics.chat.timeToFirstToken < 200 ? "default" : "secondary"}>
              {metrics.chat.timeToFirstToken}ms TTFT
            </Badge>
            <Badge variant={metrics.chat.avgResponseTime < 500 ? "default" : "secondary"}>
              {metrics.chat.avgResponseTime}ms avg response
            </Badge>
          </div>
        </CardContent>
      </Card>
 
      {/* GPT-4o Performance */}
      <Card>
        <CardHeader>
          <CardTitle>GPT-4o API</CardTitle>
        </CardHeader>
        <CardContent>
          <div className="space-y-3">
            <div>
              <div className="flex justify-between text-sm">
                <span>Requests/hour</span>
                <span>{metrics.gpt4o.requestsPerHour}</span>
              </div>
              <Progress value={Math.min(metrics.gpt4o.requestsPerHour / 100, 100)} className="mt-1" />
            </div>
            <div>
              <div className="flex justify-between text-sm">
                <span>Active streams</span>
                <span>{metrics.gpt4o.streamingActive}</span>
              </div>
            </div>
            <div className="text-sm text-muted-foreground">
              Latency: {metrics.gpt4o.avgLatency}ms
            </div>
            <div className="text-sm font-semibold text-green-600">
              ${metrics.gpt4o.costPerHour.toFixed(2)} USD/hour
            </div>
          </div>
        </CardContent>
      </Card>
 
      {/* Supabase Status */}
      <Card>
        <CardHeader>
          <CardTitle>Supabase</CardTitle>
        </CardHeader>
        <CardContent>
          <div className="space-y-2">
            <div className="flex justify-between items-center">
              <span className="text-sm">Database</span>
              <Badge variant={metrics.supabase.databaseHealth === 'healthy' ? "default" : "destructive"}>
                {metrics.supabase.databaseHealth}
              </Badge>
            </div>
            <div className="flex justify-between items-center">
              <span className="text-sm">Connections</span>
              <span className="font-semibold">{metrics.supabase.activeConnections}</span>
            </div>
            <div className="flex justify-between items-center">
              <span className="text-sm">Realtime subs</span>
              <span className="font-semibold">{metrics.supabase.realtimeSubscriptions}</span>
            </div>
            <div className="pt-2 border-t">
              <div className="text-sm text-muted-foreground">Query latency</div>
              <div className="text-lg font-bold">{metrics.supabase.queryLatency}ms</div>
            </div>
          </div>
        </CardContent>
      </Card>
 
      {/* Feature Usage */}
      <Card>
        <CardHeader>
          <CardTitle>Feature Usage</CardTitle>
        </CardHeader>
        <CardContent>
          <div className="space-y-2">
            <div className="flex justify-between">
              <span className="text-sm">Avatar Sessions</span>
              <span className="font-semibold">{metrics.features.avatarSessions}</span>
            </div>
            <div className="flex justify-between">
              <span className="text-sm">Voice Sessions</span>
              <span className="font-semibold">{metrics.features.voiceSessions}</span>
            </div>
            <div className="flex justify-between">
              <span className="text-sm">File Uploads</span>
              <span className="font-semibold">{metrics.features.fileUploads}</span>
            </div>
            <div className="pt-2 border-t">
              <div className="text-sm text-muted-foreground">Daily Limits</div>
              <div className="text-xs space-y-1">
                <div>Free: {metrics.features.dailyLimits.free.used}/{metrics.features.dailyLimits.free.limit}</div>
                <div>Pro: {metrics.features.dailyLimits.pro.used}/{metrics.features.dailyLimits.pro.limit}</div>
              </div>
            </div>
          </div>
        </CardContent>
      </Card>
    </div>
  );
}

Alert Configuration

Set up intelligent alerts for critical platform events:

Chat Performance


// lib/monitoring/alerts.ts
export interface ChatAlert {
  type: 'chat_performance';
  conditions: {
    successRate: { min: number };
    satisfaction: { min: number };
    errorRate: { max: number };
    timeToFirstToken: { max: number };
  };
  notifications: NotificationChannel[];
}
 
export const chatPerformanceAlerts: ChatAlert = {
  type: 'chat_performance',
  conditions: {
    successRate: { min: 95 },      // Alert if below 95%
    satisfaction: { min: 4.0 },    // Alert if below 4.0/5
    errorRate: { max: 2 },         // Alert if above 2%
    timeToFirstToken: { max: 500 } // Alert if TTFT > 500ms
  },
  notifications: [
    { type: 'slack', channel: '#alerts' },
    { type: 'email', recipients: ['team@earna.sh'] },
    { type: 'webhook', url: process.env.ALERT_WEBHOOK_URL }
  ]
};
 
export class AlertManager {
  async checkChatPerformance(metrics: ChatMetrics): Promise<void> {
    const alerts = [];
 
    if (metrics.chat.successRate < chatPerformanceAlerts.conditions.successRate.min) {
      alerts.push({
        type: 'low_success_rate',
        message: `Chat success rate dropped to ${metrics.chat.successRate}%`,
        severity: 'high',
        value: metrics.chat.successRate
      });
    }
 
    if (metrics.chat.timeToFirstToken > chatPerformanceAlerts.conditions.timeToFirstToken.max) {
      alerts.push({
        type: 'slow_streaming',
        message: `Time to first token increased to ${metrics.chat.timeToFirstToken}ms`,
        severity: 'medium',
        value: metrics.chat.timeToFirstToken
      });
    }
 
    if (alerts.length > 0) {
      await this.sendAlerts(alerts, chatPerformanceAlerts.notifications);
    }
  }
}

GPT-4o Monitoring


export interface GPT4oAlert {
  type: 'gpt4o_monitoring';
  conditions: {
    errorRate: { max: number };
    latency: { max: number };
    costPerHour: { max: number };
    rateLimitApproaching: { threshold: number };
  };
}
 
export const gpt4oAlerts: GPT4oAlert = {
  type: 'gpt4o_monitoring',
  conditions: {
    errorRate: { max: 1 },         // Alert if error rate > 1%
    latency: { max: 2000 },        // Alert if latency > 2s
    costPerHour: { max: 10 },      // Alert if cost > $10/hour
    rateLimitApproaching: { threshold: 80 } // Alert at 80% of rate limit
  }
};
 
export class GPT4oMonitor {
  async monitorGPT4o(metrics: PlatformMetrics): Promise<void> {
    const alerts = [];
 
    if (metrics.gpt4o.errorRate > gpt4oAlerts.conditions.errorRate.max) {
      alerts.push({
        type: 'gpt4o_error_rate',
        message: `GPT-4o error rate: ${metrics.gpt4o.errorRate}%`,
        severity: 'high'
      });
    }
 
    if (metrics.gpt4o.avgLatency > gpt4oAlerts.conditions.latency.max) {
      alerts.push({
        type: 'gpt4o_latency',
        message: `GPT-4o latency increased to ${metrics.gpt4o.avgLatency}ms`,
        severity: 'medium'
      });
    }
 
    if (metrics.gpt4o.costPerHour > gpt4oAlerts.conditions.costPerHour.max) {
      alerts.push({
        type: 'gpt4o_cost',
        message: `GPT-4o costs reached $${metrics.gpt4o.costPerHour}/hour`,
        severity: 'high'
      });
    }
 
    // Check rate limit
    const rateLimitUsage = await this.checkRateLimitUsage();
    if (rateLimitUsage > gpt4oAlerts.conditions.rateLimitApproaching.threshold) {
      alerts.push({
        type: 'rate_limit_warning',
        message: `GPT-4o rate limit at ${rateLimitUsage}%`,
        severity: 'high'
      });
    }
 
    if (alerts.length > 0) {
      await this.sendGPT4oAlerts(alerts);
    }
  }
}

Supabase Health


export interface SupabaseAlert {
  type: 'supabase_health';
  conditions: {
    connectionPool: { max: number };
    queryLatency: { max: number };
    realtimeLatency: { max: number };
    storageUsage: { max: number }; // percentage
  };
}
 
export const supabaseAlerts: SupabaseAlert = {
  type: 'supabase_health',
  conditions: {
    connectionPool: { max: 80 },    // Alert if > 80% of pool used
    queryLatency: { max: 100 },     // Alert if query latency > 100ms
    realtimeLatency: { max: 500 },  // Alert if realtime latency > 500ms
    storageUsage: { max: 90 }       // Alert if storage > 90% full
  }
};
 
export class SupabaseMonitor {
  async monitorSupabase(metrics: PlatformMetrics): Promise<void> {
    const alerts = [];
 
    if (metrics.supabase.databaseHealth !== 'healthy') {
      alerts.push({
        type: 'database_health',
        message: `Supabase database status: ${metrics.supabase.databaseHealth}`,
        severity: 'critical'
      });
    }
 
    if (metrics.supabase.queryLatency > supabaseAlerts.conditions.queryLatency.max) {
      alerts.push({
        type: 'slow_queries',
        message: `Database query latency: ${metrics.supabase.queryLatency}ms`,
        severity: 'medium'
      });
    }
 
    // Check connection pool usage
    const poolUsage = (metrics.supabase.activeConnections / 20) * 100;
    if (poolUsage > supabaseAlerts.conditions.connectionPool.max) {
      alerts.push({
        type: 'connection_pool_high',
        message: `Connection pool at ${poolUsage.toFixed(1)}% capacity`,
        severity: 'high'
      });
    }
 
    if (alerts.length > 0) {
      await this.sendSupabaseAlerts(alerts);
    }
  }
}

System Health


export interface SystemHealthAlert {
  type: 'system_health';
  conditions: {
    uptime: { min: number };
    apiLatency: { max: number };
    errorRate: { max: number };
    cacheHitRate: { min: number };
  };
}
 
export const systemHealthAlerts: SystemHealthAlert = {
  type: 'system_health',
  conditions: {
    uptime: { min: 99.9 },         // Alert if uptime < 99.9%
    apiLatency: { max: 500 },      // Alert if API latency > 500ms
    errorRate: { max: 0.5 },       // Alert if error rate > 0.5%
    cacheHitRate: { min: 70 }      // Alert if cache hit < 70%
  }
};
 
export class SystemHealthMonitor {
  async monitorSystemHealth(metrics: PlatformMetrics): Promise<void> {
    const alerts = [];
 
    if (metrics.system.uptime < systemHealthAlerts.conditions.uptime.min) {
      alerts.push({
        type: 'uptime_low',
        message: `System uptime dropped to ${metrics.system.uptime}%`,
        severity: 'critical'
      });
    }
 
    if (metrics.system.apiLatency > systemHealthAlerts.conditions.apiLatency.max) {
      alerts.push({
        type: 'api_latency_high',
        message: `API latency increased to ${metrics.system.apiLatency}ms`,
        severity: 'medium'
      });
    }
 
    if (metrics.system.cacheHitRate < systemHealthAlerts.conditions.cacheHitRate.min) {
      alerts.push({
        type: 'cache_hit_low',
        message: `Cache hit rate: ${metrics.system.cacheHitRate}%`,
        severity: 'low'
      });
    }
 
    if (alerts.length > 0) {
      await this.sendSystemAlerts(alerts);
    }
  }
}

User Analytics

Chat Analytics

Track detailed chat patterns and user behavior:


// lib/analytics/chat-analytics.ts
export interface ChatAnalytics {
  sessionMetrics: {
    duration: number;
    messageCount: number;
    modelSwitches: number;      // Times user switched models
    userSatisfaction: number;
    completionReason: 'completed' | 'abandoned' | 'limit_reached' | 'error';
  };
  modelUsage: {
    primary: string;             // Most used model
    models: Array<{
      model: string;
      messages: number;
      tokens: number;
      cost: number;
    }>;
  };
  featureUsage: {
    avatarUsed: boolean;
    voiceUsed: boolean;
    filesUploaded: number;
    codeExecuted: boolean;
  };
  performance: {
    avgResponseTime: number;
    avgTimeToFirstToken: number;
    streamingQuality: number;   // 0-100 score
  };
}
 
export class ChatTracker {
  async trackChat(
    chatId: string,
    analytics: ChatAnalytics
  ): Promise<void> {
    await supabase.from('chat_analytics').insert({
      chat_id: chatId,
      session_duration: analytics.sessionMetrics.duration,
      message_count: analytics.sessionMetrics.messageCount,
      model_switches: analytics.sessionMetrics.modelSwitches,
      user_satisfaction: analytics.sessionMetrics.userSatisfaction,
      completion_reason: analytics.sessionMetrics.completionReason,
      model_usage: analytics.modelUsage,
      feature_usage: analytics.featureUsage,
      performance_metrics: analytics.performance,
      created_at: new Date()
    });
 
    // Update real-time analytics
    await this.updateRealTimeAnalytics(analytics);
  }
 
  async generateInsights(timeframe: 'day' | 'week' | 'month'): Promise<any> {
    const { data } = await supabase
      .from('chat_analytics')
      .select('*')
      .gte('created_at', this.getTimeframeStart(timeframe));
 
    return {
      totalChats: data.length,
      avgDuration: this.average(data.map(d => d.session_duration)),
      avgSatisfaction: this.average(data.map(d => d.user_satisfaction)),
      modelPreference: this.getMostUsedModel(data),
      featureAdoption: this.calculateFeatureAdoption(data),
      performanceTrends: this.calculatePerformanceTrends(data)
    };
  }
}

User Journey Tracking

Monitor user progression through the platform:


// lib/analytics/user-journey.ts
export interface UserJourney {
  stages: {
    signup: Date | null;         // Account created
    firstChat: Date | null;      // First message sent
    modelExploration: Date | null; // Tried alternative models
    featureAdoption: Date | null;  // Used advanced features
    subscription: Date | null;    // Upgraded to pro
    retention: Date | null;      // Active after 30 days
  };
  metrics: {
    timeToFirstChat: number;     // Minutes from signup
    timeToSubscription: number;  // Days to upgrade
    engagementScore: number;     // 0-100
    retentionProbability: number; // 0-1
  };
}
 
export class UserJourneyTracker {
  async updateUserStage(userId: string, stage: keyof UserJourney['stages']): Promise<void> {
    const { error } = await supabase
      .from('user_journeys')
      .upsert({
        user_id: userId,
        [stage]: new Date(),
        updated_at: new Date()
      });
 
    if (!error) {
      await this.calculateJourneyMetrics(userId);
    }
  }
 
  async getJourneyInsights(): Promise<any> {
    const { data } = await supabase
      .from('user_journeys')
      .select('*')
      .not('retention', 'is', null); // Only users who reached retention
 
    return {
      avgTimeToFirstChat: this.calculateAverageTime(data, 'firstChat'),
      conversionRates: {
        signupToFirstChat: this.calculateConversionRate(data, 'signup', 'firstChat'),
        firstChatToExploration: this.calculateConversionRate(data, 'firstChat', 'modelExploration'),
        explorationToSubscription: this.calculateConversionRate(data, 'modelExploration', 'subscription')
      },
      retentionRate: this.calculateRetentionRate(data)
    };
  }
}

Cost Monitoring

AI Model Cost Optimization

Track and optimize AI model costs:


// lib/monitoring/cost-monitoring.ts
export class CostMonitor {
  private readonly MODEL_COSTS = {
    'gpt-4o': { input: 0.005, output: 0.015 },      // per 1K tokens
    'claude-3-opus': { input: 0.015, output: 0.075 },
    'gemini-1.5-pro': { input: 0.00125, output: 0.005 },
    'mistral-large': { input: 0.004, output: 0.012 }
  };
 
  async trackModelUsage(
    model: string,
    inputTokens: number,
    outputTokens: number,
    responseTime: number
  ): Promise<void> {
    const cost = this.calculateCost(model, inputTokens, outputTokens);
    
    await supabase.from('model_usage').insert({
      model,
      input_tokens: inputTokens,
      output_tokens: outputTokens,
      cost,
      response_time: responseTime,
      created_at: new Date()
    });
 
    // Check cost limits
    await this.checkCostLimits();
  }
 
  private calculateCost(model: string, inputTokens: number, outputTokens: number): number {
    const modelCost = this.MODEL_COSTS[model as keyof typeof this.MODEL_COSTS];
    if (!modelCost) return 0;
 
    return (inputTokens * modelCost.input + outputTokens * modelCost.output) / 1000;
  }
 
  async getCostSummary(period: 'hour' | 'day' | 'week' | 'month'): Promise<any> {
    const timeWindow = this.getTimeWindow(period);
    
    const { data } = await supabase
      .from('model_usage')
      .select('model, cost, input_tokens, output_tokens')
      .gte('created_at', timeWindow);
 
    const summary = data.reduce((acc, usage) => {
      if (!acc[usage.model]) {
        acc[usage.model] = { cost: 0, requests: 0, tokens: 0 };
      }
      acc[usage.model].cost += usage.cost;
      acc[usage.model].requests++;
      acc[usage.model].tokens += usage.input_tokens + usage.output_tokens;
      return acc;
    }, {});
 
    return {
      totalCost: Object.values(summary).reduce((sum: number, m: any) => sum + m.cost, 0),
      costByModel: summary,
      projectedMonthlyCost: this.projectMonthlyCost(summary, period)
    };
  }
 
  private async checkCostLimits(): Promise<void> {
    const hourlyCost = await this.getCostSummary('hour');
    const dailyCost = await this.getCostSummary('day');
 
    const COST_LIMITS = {
      hourly: 50,    // $50/hour
      daily: 500,    // $500/day
      monthly: 10000 // $10,000/month
    };
 
    if (hourlyCost.totalCost > COST_LIMITS.hourly) {
      await this.sendCostAlert('hourly', hourlyCost.totalCost, COST_LIMITS.hourly);
    }
 
    if (dailyCost.totalCost > COST_LIMITS.daily) {
      await this.sendCostAlert('daily', dailyCost.totalCost, COST_LIMITS.daily);
    }
  }
}

Performance Optimization

Model Selection Optimizer

Intelligently select the best model based on current conditions:


// lib/monitoring/performance-optimizer.ts
export class PerformanceOptimizer {
  private cache = new Map<string, any>();
 
  async selectOptimalModel(query: string, context: any): Promise<string> {
    // Check cache first
    const cacheKey = this.generateCacheKey(query, context);
    const cached = this.cache.get(cacheKey);
    if (cached && this.isCacheValid(cached)) {
      return cached.model;
    }
 
    // Get current model performance
    const modelPerformance = await this.getModelPerformanceMetrics();
    
    // Select optimal model
    const optimalModel = this.selectBestModel(query, context, modelPerformance);
    
    // Cache the decision
    this.cache.set(cacheKey, {
      model: optimalModel,
      timestamp: Date.now(),
      performance: modelPerformance
    });
 
    return optimalModel;
  }
 
  private async getModelPerformanceMetrics(): Promise<any> {
    const timeWindow = 15 * 60 * 1000; // Last 15 minutes
    
    return {
      'gpt-4o': {
        avgLatency: await this.getModelLatency('gpt-4o', timeWindow),
        errorRate: await this.getModelErrorRate('gpt-4o', timeWindow),
        availability: await this.checkModelAvailability('gpt-4o'),
        cost: 0.005 // per 1K tokens
      },
      'claude-3-opus': {
        avgLatency: await this.getModelLatency('claude-3-opus', timeWindow),
        errorRate: await this.getModelErrorRate('claude-3-opus', timeWindow),
        availability: await this.checkModelAvailability('claude-3-opus'),
        cost: 0.015
      },
      'gemini-1.5-pro': {
        avgLatency: await this.getModelLatency('gemini-1.5-pro', timeWindow),
        errorRate: await this.getModelErrorRate('gemini-1.5-pro', timeWindow),
        availability: await this.checkModelAvailability('gemini-1.5-pro'),
        cost: 0.00125
      }
    };
  }
 
  private selectBestModel(query: string, context: any, performance: any): string {
    // Default to GPT-4o if no alternatives available
    if (!performance['claude-3-opus'].availability && !performance['gemini-1.5-pro'].availability) {
      return 'gpt-4o';
    }
 
    // Score each model
    const scores = Object.entries(performance).map(([model, metrics]: [string, any]) => {
      if (!metrics.availability) return { model, score: -1 };
      
      let score = 100;
      
      // Penalize high latency
      score -= metrics.avgLatency / 100;
      
      // Penalize high error rates
      score -= metrics.errorRate * 10;
      
      // Consider cost vs performance
      if (context.prioritizeCost) {
        score -= metrics.cost * 100;
      }
      
      // Model-specific optimizations
      if (query.length > 10000 && model === 'gemini-1.5-pro') {
        score += 20; // Gemini handles long context well
      }
      
      if (context.requiresVision && model === 'gpt-4o') {
        score += 25; // GPT-4o has best vision capabilities
      }
 
      return { model, score };
    });
 
    // Return highest scoring available model
    return scores
      .filter(s => s.score > 0)
      .sort((a, b) => b.score - a.score)[0]?.model || 'gpt-4o';
  }
}

Observability Tools

Integration with External Services

Vercel Analytics


// lib/monitoring/vercel-analytics.ts
import { track } from '@vercel/analytics';
 
export class VercelAnalytics {
  trackChatStarted(userId: string, model: string): void {
    track('chat_started', {
      userId,
      model,
      timestamp: new Date().toISOString()
    });
  }
 
  trackMessageSent(userId: string, model: string, tokens: number): void {
    track('message_sent', {
      userId,
      model,
      tokens,
      timestamp: new Date().toISOString()
    });
  }
 
  trackModelSwitched(userId: string, fromModel: string, toModel: string): void {
    track('model_switched', {
      userId,
      fromModel,
      toModel,
      timestamp: new Date().toISOString()
    });
  }
 
  trackFeatureUsed(userId: string, feature: string): void {
    track('feature_used', {
      userId,
      feature,
      timestamp: new Date().toISOString()
    });
  }
 
  trackSubscriptionUpgrade(userId: string, tier: string): void {
    track('subscription_upgraded', {
      userId,
      tier,
      timestamp: new Date().toISOString()
    });
  }
}

Sentry Error Tracking


// lib/monitoring/sentry.ts
import * as Sentry from '@sentry/nextjs';
 
export class ErrorMonitoring {
  static initialize(): void {
    Sentry.init({
      dsn: process.env.SENTRY_DSN,
      environment: process.env.NODE_ENV,
      tracesSampleRate: 1.0,
      beforeSend(event) {
        // Filter out sensitive information
        if (event.request?.data) {
          event.request.data = '[Filtered]';
        }
        return event;
      }
    });
  }
 
  static captureChatError(error: Error, context: any): void {
    Sentry.withScope(scope => {
      scope.setTag('error_type', 'chat');
      scope.setContext('chat', {
        chatId: context.chatId,
        userId: context.userId,
        model: context.model
      });
      Sentry.captureException(error);
    });
  }
 
  static captureModelError(error: Error, model: string, query: string): void {
    Sentry.withScope(scope => {
      scope.setTag('error_type', 'ai_model');
      scope.setContext('ai_request', {
        model,
        queryLength: query.length,
        timestamp: new Date().toISOString()
      });
      Sentry.captureException(error);
    });
  }
 
  static captureSupabaseError(error: Error, operation: string): void {
    Sentry.withScope(scope => {
      scope.setTag('error_type', 'supabase');
      scope.setContext('database', {
        operation,
        timestamp: new Date().toISOString()
      });
      Sentry.captureException(error);
    });
  }
}

PostHog Analytics


// lib/monitoring/posthog.ts
import { PostHog } from 'posthog-node';
 
export class PostHogAnalytics {
  private client: PostHog;
 
  constructor() {
    this.client = new PostHog(
      process.env.POSTHOG_API_KEY!,
      { host: 'https://app.posthog.com' }
    );
  }
 
  trackUserJourney(userId: string, stage: string, properties: any): void {
    this.client.capture({
      distinctId: userId,
      event: 'user_journey_stage',
      properties: {
        stage,
        ...properties,
        timestamp: new Date().toISOString()
      }
    });
  }
 
  trackFeatureAdoption(userId: string, feature: string, success: boolean): void {
    this.client.capture({
      distinctId: userId,
      event: 'feature_adopted',
      properties: {
        feature,
        success,
        timestamp: new Date().toISOString()
      }
    });
  }
 
  trackModelPerformance(model: string, latency: number, tokens: number): void {
    this.client.capture({
      distinctId: 'system',
      event: 'model_performance',
      properties: {
        model,
        latency,
        tokens,
        timestamp: new Date().toISOString()
      }
    });
  }
 
  async shutdown(): Promise<void> {
    await this.client.shutdown();
  }
}

Comprehensive monitoring helps ensure optimal performance and user experience. Set up alerts for critical metrics and regularly review analytics to identify improvement opportunities.

Best Practices

Monitoring Checklist

Real-time Dashboards: Monitor key metrics continuously
Intelligent Alerts: Set up proactive alerting for critical issues
Cost Tracking: Monitor AI model costs and optimize usage
User Analytics: Track user journeys and satisfaction
Performance Monitoring: Optimize response times and streaming quality
Error Tracking: Comprehensive error monitoring with Sentry
Database Health: Monitor Supabase connections and performance
Feature Adoption: Track usage of avatars, voice, and file uploads

Monitoring Strategy

Proactive Monitoring: Detect issues before users are affected
User-Centric Metrics: Focus on metrics that impact user experience
Cost Optimization: Balance performance with cost efficiency
Continuous Improvement: Use analytics to drive product improvements
Privacy-First: Ensure all monitoring respects user privacy

For monitoring support and custom dashboards, contact support@earna.sh.