CoT 없이 성능 올리는 Self-Consistency 구현법

type Candidate = {
  answer: string;
  confidence?: number; // 모델이 숫자로 내주면 가중치에 활용 가능
};

function normalizeAnswer(raw: string): string {
  return raw
    .trim()
    .toLowerCase()
    .replace(/\s+/g, " ")
    .replace(/["'`]/g, "")
    .replace(/[.,!?]/g, "");
}

function majorityVote(candidates: Candidate[]): { final: Candidate; stats: Record<string, number> } {
  const counts: Record<string, number> = {};
  const rep: Record<string, Candidate> = {};

  for (const c of candidates) {
    const key = normalizeAnswer(c.answer);
    counts[key] = (counts[key] ?? 0) + 1;
    rep[key] = rep[key] ?? c;
  }

  const sorted = Object.entries(counts).sort((a, b) => b[1] - a[1]);
  const winnerKey = sorted[0]?.[0];
  if (!winnerKey) throw new Error("No candidates");

  return { final: rep[winnerKey], stats: counts };
}

async function sampleOnce(prompt: string): Promise<Candidate> {
  // 예시: OpenAI 호환 Chat Completions API
  // 실제 SDK/엔드포인트는 사용 환경에 맞게 바꾸세요.
  const res = await fetch("/api/llm", {
    method: "POST",
    headers: { "content-type": "application/json" },
    body: JSON.stringify({
      model: "gpt-4.1-mini",
      temperature: 0.9,
      top_p: 0.95,
      messages: [
        {
          role: "system",
          content:
            "You are a precise assistant. Do not reveal chain-of-thought. Output only valid JSON.",
        },
        {
          role: "user",
          content:
            `${prompt}\n\nReturn JSON with keys: answer (string), confidence (number 0..1).`,
        },
      ],
      response_format: { type: "json_object" },
    }),
  });

  const data = await res.json();
  // data parsing은 실제 응답 구조에 맞게 조정
  const content = data.choices[0].message.content;
  const parsed = JSON.parse(content);

  return {
    answer: String(parsed.answer ?? "").trim(),
    confidence: typeof parsed.confidence === "number" ? parsed.confidence : undefined,
  };
}

export async function selfConsistentAnswer(prompt: string, n = 7) {
  const candidates: Candidate[] = [];
  for (let i = 0; i < n; i++) {
    candidates.push(await sampleOnce(prompt));
  }

  const { final, stats } = majorityVote(candidates);
  return { final, candidates, stats };
}

type Label = "YES" | "NO" | "UNKNOWN";

function normalizeLabel(x: string): Label {
  const v = x.trim().toUpperCase();
  if (v === "YES" || v === "Y") return "YES";
  if (v === "NO" || v === "N") return "NO";
  return "UNKNOWN";
}

function voteLabel(labels: string[]): { label: Label; counts: Record<Label, number> } {
  const counts: Record<Label, number> = { YES: 0, NO: 0, UNKNOWN: 0 };
  for (const l of labels) counts[normalizeLabel(l)]++;

  const label = (Object.entries(counts).sort((a, b) => b[1] - a[1])[0][0] as Label) ?? "UNKNOWN";
  return { label, counts };
}

function weightedVote(candidates: Candidate[]) {
  const score: Record<string, number> = {};
  const rep: Record<string, Candidate> = {};

  for (const c of candidates) {
    const key = normalizeAnswer(c.answer);
    const w = typeof c.confidence === "number" ? Math.max(0, Math.min(1, c.confidence)) : 1;
    score[key] = (score[key] ?? 0) + w;
    rep[key] = rep[key] ?? c;
  }

  const winnerKey = Object.entries(score).sort((a, b) => b[1] - a[1])[0]?.[0];
  if (!winnerKey) throw new Error("No candidates");

  return { final: rep[winnerKey], score };
}

async function verifyAnswer(question: string, answer: string): Promise<boolean> {
  const res = await fetch("/api/llm", {
    method: "POST",
    headers: { "content-type": "application/json" },
    body: JSON.stringify({
      model: "gpt-4.1-mini",
      temperature: 0,
      messages: [
        {
          role: "system",
          content:
            "You are a strict verifier. Do not reveal chain-of-thought. Output only JSON.",
        },
        {
          role: "user",
          content:
            `Question: ${question}\nAnswer: ${answer}\n\nReturn JSON: {"valid": true|false}.`,
        },
      ],
      response_format: { type: "json_object" },
    }),
  });

  const data = await res.json();
  const parsed = JSON.parse(data.choices[0].message.content);
  return Boolean(parsed.valid);
}

export async function selfConsistencyAdaptive(prompt: string, maxN = 9) {
  const candidates: Candidate[] = [];
  const counts: Record<string, number> = {};

  for (let i = 0; i < maxN; i++) {
    const c = await sampleOnce(prompt);
    candidates.push(c);

    const key = normalizeAnswer(c.answer);
    counts[key] = (counts[key] ?? 0) + 1;

    // 조기 종료: 2표 이상이면서 나머지로 뒤집기 어려운 경우 등
    const top = Object.values(counts).sort((a, b) => b - a);
    const best = top[0] ?? 0;
    const second = top[1] ?? 0;

    if (best >= 3 && best - second >= 2) break;
  }

  const { final, stats } = majorityVote(candidates);
  return { final, candidates, stats };
}

{"label":"A","answer":"..."}