OpenAI API 429 Rate Limit 재시도·큐잉 설계

import OpenAI from "openai";

const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY! });

function sleep(ms: number) {
  return new Promise((r) => setTimeout(r, ms));
}

function parseRetryAfterSeconds(err: any): number | null {
  const v = err?.response?.headers?.["retry-after"] ?? err?.headers?.["retry-after"];
  if (!v) return null;
  const n = Number(v);
  return Number.isFinite(n) ? n : null;
}

function jitteredBackoffMs(attempt: number, baseMs = 250, capMs = 10_000) {
  // full jitter: random(0, min(cap, base * 2^attempt))
  const max = Math.min(capMs, baseMs * Math.pow(2, attempt));
  return Math.floor(Math.random() * max);
}

async function callWithRetry<T>(fn: () => Promise<T>, opts?: { maxRetries?: number }) {
  const maxRetries = opts?.maxRetries ?? 6;

  for (let attempt = 0; ; attempt++) {
    try {
      return await fn();
    } catch (err: any) {
      const status = err?.status ?? err?.response?.status;
      const is429 = status === 429;

      if (!is429) throw err;
      if (attempt >= maxRetries) throw err;

      const ra = parseRetryAfterSeconds(err);
      const waitMs = ra != null ? ra * 1000 : jitteredBackoffMs(attempt);
      await sleep(waitMs);
    }
  }
}

export async function generateText(prompt: string) {
  return callWithRetry(async () => {
    return client.responses.create({
      model: "gpt-4.1-mini",
      input: prompt,
    });
  });
}

class Semaphore {
  private available: number;
  private waiters: Array<() => void> = [];

  constructor(private capacity: number) {
    this.available = capacity;
  }

  async acquire() {
    if (this.available > 0) {
      this.available -= 1;
      return;
    }
    await new Promise<void>((resolve) => this.waiters.push(resolve));
  }

  release() {
    const w = this.waiters.shift();
    if (w) w();
    else this.available = Math.min(this.capacity, this.available + 1);
  }
}

const sem = new Semaphore(5); // 동시에 5개만

async function withConcurrencyLimit<T>(fn: () => Promise<T>) {
  await sem.acquire();
  try {
    return await fn();
  } finally {
    sem.release();
  }
}

export async function generateTextLimited(prompt: string) {
  return withConcurrencyLimit(() =>
    callWithRetry(() =>
      client.responses.create({ model: "gpt-4.1-mini", input: prompt })
    )
  );
}

import Bottleneck from "bottleneck";
import OpenAI from "openai";

const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY! });

// minTime: 요청 간 최소 간격(ms)
// maxConcurrent: 동시 실행 제한
const limiter = new Bottleneck({
  minTime: 120,      // 대략 초당 8.3 req
  maxConcurrent: 4,
});

async function createResponse(input: string) {
  return limiter.schedule(() =>
    callWithRetry(() =>
      client.responses.create({
        model: "gpt-4.1-mini",
        input,
      })
    )
  );
}

type Job<T> = {
  cost: number; // 예상 토큰 비용
  run: () => Promise<T>;
  resolve: (v: T) => void;
  reject: (e: any) => void;
};

class TokenBudgetQueue {
  private queue: Job<any>[] = [];
  private used = 0;
  private windowStart = Date.now();

  constructor(private budgetPerMinute: number) {
    setInterval(() => this.tick(), 50).unref();
  }

  private resetWindowIfNeeded() {
    const now = Date.now();
    if (now - this.windowStart >= 60_000) {
      this.windowStart = now;
      this.used = 0;
    }
  }

  private async tick() {
    this.resetWindowIfNeeded();
    const job = this.queue[0];
    if (!job) return;

    if (this.used + job.cost > this.budgetPerMinute) return;

    this.queue.shift();
    this.used += job.cost;

    try {
      const v = await job.run();
      job.resolve(v);
    } catch (e) {
      job.reject(e);
    }
  }

  schedule<T>(cost: number, run: () => Promise<T>) {
    return new Promise<T>((resolve, reject) => {
      this.queue.push({ cost, run, resolve, reject });
    });
  }
}

const tokenQueue = new TokenBudgetQueue(120_000); // 분당 120k 토큰 예산(예시)

import OpenAI from "openai";

const client = new OpenAI({ apiKey: process.env.OPENAI_API_KEY! });

function roughTokenEstimate(input: string, maxOutputTokens: number) {
  // 매우 러프한 근사: 영어는 4 chars/token 근처, 한국어는 더 보수적으로 잡는 편
  const inputTokens = Math.ceil(input.length / 3);
  return inputTokens + maxOutputTokens;
}

export async function queuedResponse(input: string) {
  const maxOutputTokens = 800;
  const cost = roughTokenEstimate(input, maxOutputTokens);

  return tokenQueue.schedule(cost, () =>
    callWithRetry(() =>
      client.responses.create({
        model: "gpt-4.1-mini",
        input,
        max_output_tokens: maxOutputTokens,
      })
    )
  );
}