prompt-optimization

# 提示词优化指南 ## 系统性改进流程 ### 1. 建立基准 python def establish_baseline(prompt, test_cases): results = { 'accuracy': 0, 'avg_tokens': 0, 'avg_latency': 0, 'success_rate': 0 } for test_case in test_cases: response = llm.complete(prompt.format(**test_case['input'])) results['accuracy'] += evaluate_accuracy(response, test_case['expected']) results['avg_tokens'] += count_tokens(response) results['avg_latency'] += measure_latency(response) results['success_rate'] += is_valid_response(response) # 计算测试用例的平均值 n = len(test_cases) return {k: v/n for k, v in results.items()} ### 2. 迭代改进工作流 初始提示词 → 测试 → 分析失败原因 → 改进 → 测试 → 重复 python class PromptOptimizer: def __init__(self, initial_prompt, test_suite): self.prompt = initial_prompt self.test_suite = test_suite self.history = [] def optimize(self, max_iterations=10): for i in range(max_iterations): # 测试当前提示词 results = self.evaluate_prompt(self.prompt) self.history.append({ 'iteration': i, 'prompt': self.prompt, 'results': results }) # 如果足够好则停止 if results['accuracy'] > 0.95: break # 分析失败原因 failures = self.analyze_failures(results) # 生成改进建议 refinements = self.generate_refinements(failures) # 应用最佳改进 self.prompt = self.select_best_refinement(refinements) return self.get_best_prompt() ### 3. A/B 测试框架 python class PromptABTest: def __init__(self, variant_a, variant_b): self.variant_a = variant_a self.variant_b = variant_b def run_test(self, test_queries, metrics=['accuracy', 'latency']): results = { 'A': {m: [] for m in metrics}, 'B': {m: [] for m in metrics} } for query in test_queries: # 随机分配变体 (50/50 分割) variant = 'A' if random.random() < 0.5 else 'B' prompt = self.variant_a if variant == 'A' else self.variant_b response, metrics_data = self.execute_with_metrics( prompt.format(query=qu