Cost Optimization for LLM Applications: Spending Wisely on AI

type Complexity = 'simple' | 'medium' | 'complex';

function selectModel(task: Task): string {
  const complexity = assessComplexity(task);
  
  return {
    simple: 'claude-3-haiku',      // $0.25/M - 20x cheaper
    medium: 'gpt-4o-mini',         // $0.15/M 
    complex: 'claude-3-opus'       // $15/M - only when needed
  }[complexity];
}

// Automatic complexity detection
function assessComplexity(task: Task): Complexity {
  if (task.type === 'classification') return 'simple';
  if (task.type === 'generation' && task.length < 100) return 'simple';
  if (task.requiresReasoning) return 'complex';
  return 'medium';
}

import { createHash } from 'crypto';

const cache = new Map<string, CachedResponse>();

async function cachedComplete(prompt: string, options: Options) {
  const key = createHash('sha256')
    .update(JSON.stringify({ prompt, ...options }))
    .digest('hex');
  
  // Check cache
  const cached = cache.get(key);
  if (cached && !isExpired(cached)) {
    return cached.response;
  }
  
  // Call API
  const response = await llm.complete(prompt, options);
  
  // Cache if deterministic
  if (options.temperature === 0) {
    cache.set(key, { response, timestamp: Date.now() });
  }
  
  return response;
}

// Before: 500 tokens
const verbose = `
  Please analyze the following text carefully and thoroughly.
  Consider all aspects of the content including but not limited to
  the main themes, supporting arguments, and overall structure.
  
  Text: ${content}
`;

// After: 50 tokens
const compressed = `Analyze:\n${content}`;

// Same quality, 90% fewer tokens

// Expensive: 10 API calls
async function processItems(items: Item[]) {
  return Promise.all(items.map(item => 
    llm.complete(`Classify: ${item.text}`)
  ));
}

// Cheap: 1 API call
async function batchProcess(items: Item[]) {
  const prompt = items.map((item, i) => 
    `${i+1}. ${item.text}`
  ).join('\n');
  
  const result = await llm.complete(
    `Classify each item (positive/negative):\n${prompt}\n\nRespond with numbers and classifications only.`
  );
  
  return parseClassifications(result);
}

// Expensive: Unlimited output
const verbose = await llm.complete(prompt);
// Often produces 500+ tokens

// Cheap: Controlled output
const concise = await llm.complete(prompt, {
  max_tokens: 100,
  stop: ['\n\n']  // Stop at paragraph break
});

async function optimizedComplete(prompt: string) {
  // Try cheap model first
  const cheapResult = await cheapModel.complete(prompt);
  
  // Validate quality
  if (await isHighQuality(cheapResult)) {
    return cheapResult;
  }
  
  // Fallback to expensive only if needed
  return expensiveModel.complete(prompt);
}

class CostTracker {
  private costs: CostEntry[] = [];
  
  track(model: string, inputTokens: number, outputTokens: number) {
    const cost = this.calculateCost(model, inputTokens, outputTokens);
    this.costs.push({ model, cost, timestamp: new Date() });
    
    // Alert on threshold
    if (this.dailyCost() > this.dailyBudget) {
      this.alert('Daily budget exceeded');
    }
  }
  
  report(): CostReport {
    return {
      today: this.dailyCost(),
      thisMonth: this.monthlyCost(),
      byModel: this.costByModel(),
      trend: this.costTrend()
    };
  }
}