OpenAI 429 Rate Limit 대응 - 백오프·배치·캐시

type RetryOptions = {
  maxRetries: number;
  baseDelayMs: number;
  maxDelayMs: number;
  timeoutMs: number;
};

function sleep(ms: number) {
  return new Promise((r) => setTimeout(r, ms));
}

function parseRetryAfterMs(headers: Headers): number | null {
  const v = headers.get("retry-after");
  if (!v) return null;

  // seconds 형태가 일반적
  const seconds = Number(v);
  if (Number.isFinite(seconds)) return Math.max(0, seconds * 1000);

  // 날짜 형태도 가능
  const dateMs = Date.parse(v);
  if (!Number.isNaN(dateMs)) return Math.max(0, dateMs - Date.now());

  return null;
}

async function withRetry<T>(fn: () => Promise<T>, opts: RetryOptions): Promise<T> {
  const started = Date.now();

  for (let attempt = 0; attempt <= opts.maxRetries; attempt++) {
    try {
      return await fn();
    } catch (e: any) {
      const elapsed = Date.now() - started;
      const isLast = attempt === opts.maxRetries;

      // 여기서는 예시로 e.status를 본다고 가정
      const status = e?.status;
      const headers: Headers | undefined = e?.headers;

      const retryable = status === 429 || status === 503 || status === 502;
      if (!retryable || isLast || elapsed > opts.timeoutMs) throw e;

      const retryAfterMs = headers ? parseRetryAfterMs(headers) : null;

      // 지수 백오프: base * 2^attempt
      const exp = opts.baseDelayMs * Math.pow(2, attempt);
      const capped = Math.min(opts.maxDelayMs, exp);

      // 지터: 0.5x ~ 1.0x
      const jittered = Math.floor(capped * (0.5 + Math.random() * 0.5));

      const delay = retryAfterMs !== null ? Math.max(retryAfterMs, jittered) : jittered;
      await sleep(delay);
    }
  }

  // 논리상 도달하지 않음
  throw new Error("unreachable");
}

type Task = {
  id: string;
  input: string;
  resolve: (v: string) => void;
  reject: (e: unknown) => void;
};

class MicroBatcher {
  private queue: Task[] = [];
  private timer: NodeJS.Timeout | null = null;

  constructor(private windowMs: number, private maxBatchSize: number) {}

  enqueue(id: string, input: string): Promise<string> {
    return new Promise((resolve, reject) => {
      this.queue.push({ id, input, resolve, reject });

      if (this.queue.length >= this.maxBatchSize) {
        this.flush().catch(() => void 0);
        return;
      }

      if (!this.timer) {
        this.timer = setTimeout(() => {
          this.flush().catch(() => void 0);
        }, this.windowMs);
      }
    });
  }

  private async flush() {
    if (this.timer) {
      clearTimeout(this.timer);
      this.timer = null;
    }

    const batch = this.queue.splice(0, this.maxBatchSize);
    if (batch.length === 0) return;

    try {
      // 입력을 구조화해서 한 번에 호출
      const payload = batch.map((t) => ({ id: t.id, input: t.input }));

      const responseText = await callLLM(payload);

      // LLM 응답을 JSON으로 받는 설계(아래 프롬프트 참고)
      const parsed = JSON.parse(responseText) as { id: string; output: string }[];
      const map = new Map(parsed.map((p) => [p.id, p.output]));

      for (const t of batch) {
        const out = map.get(t.id);
        if (!out) t.reject(new Error("missing output"));
        else t.resolve(out);
      }
    } catch (e) {
      for (const t of batch) t.reject(e);
    }
  }
}

async function callLLM(items: { id: string; input: string }[]): Promise<string> {
  // 실제로는 OpenAI Responses API 호출
  // 여기서는 예시로 프롬프트만 설명
  const prompt = {
    instruction:
      "You are a JSON generator. Return ONLY JSON array. Each element: {id, output}.",
    items,
  };

  // fetch 또는 SDK 호출 후 text 반환
  return JSON.stringify(items.map((x) => ({ id: x.id, output: x.input.toUpperCase() })));
}

import crypto from "crypto";

type CacheKeyInput = {
  model: string;
  promptVersion: string;
  system: string;
  user: string;
  temperature: number;
  maxOutputTokens: number;
};

export function makeCacheKey(x: CacheKeyInput) {
  const raw = JSON.stringify(x);
  return crypto.createHash("sha256").update(raw).digest("hex");
}

import { createClient } from "redis";

const redis = createClient({ url: process.env.REDIS_URL });
await redis.connect();

async function cachedLLM(key: string, ttlSec: number, fn: () => Promise<string>) {
  const hit = await redis.get(key);
  if (hit) return hit;

  try {
    const v = await fn();
    await redis.set(key, v, { EX: ttlSec });
    return v;
  } catch (e: any) {
    // 429는 짧게 음수 캐시해서 동시 폭주를 완화
    if (e?.status === 429) {
      await redis.set(key + ":err429", "1", { EX: 10 });
    }
    throw e;
  }
}

async function handleRequest(req: { userText: string }) {
  const key = makeCacheKey({
    model: "gpt-4.1-mini",
    promptVersion: "v3",
    system: "classifier",
    user: req.userText,
    temperature: 0,
    maxOutputTokens: 200,
  });

  const cached = await redis.get(key);
  if (cached) return cached;

  const result = await batcher.enqueue(key, req.userText);

  await redis.set(key, result, { EX: 3600 });
  return result;
}

// batcher 내부 callLLM은 withRetry로 감싸서 429에 대응