Gemini API 429 쿼터·레이트리밋 재시도 설계

type RetryDecision = {
  retry: boolean;
  reason: string;
  retryAfterMs?: number;
};

function parseRetryAfterMs(headers: Headers): number | undefined {
  const v = headers.get('retry-after');
  if (!v) return;
  const sec = Number(v);
  if (!Number.isFinite(sec)) return;
  return Math.max(0, sec * 1000);
}

function isLikelyQuotaError(message: string): boolean {
  const m = message.toLowerCase();
  return m.includes('quota') || m.includes('exceeded your quota') || m.includes('billing');
}

function isLikelyRateLimitError(message: string): boolean {
  const m = message.toLowerCase();
  return m.includes('rate') || m.includes('too many') || m.includes('requests');
}

function decideRetry429(params: {
  status: number;
  message: string;
  retryAfterMs?: number;
}): RetryDecision {
  const { status, message, retryAfterMs } = params;
  if (status !== 429) return { retry: false, reason: 'not-429' };

  // 쿼터 소진 가능성이 높으면 재시도 금지
  if (isLikelyQuotaError(message)) {
    return { retry: false, reason: 'quota-exceeded' };
  }

  // Retry-After가 있으면 단기 제한일 가능성이 높으므로 존중
  if (typeof retryAfterMs === 'number') {
    return { retry: true, reason: 'rate-limited-retry-after', retryAfterMs };
  }

  // 메시지가 rate limit 류로 보이면 제한적으로 재시도
  if (isLikelyRateLimitError(message)) {
    return { retry: true, reason: 'rate-limited-heuristic' };
  }

  // 애매하면 재시도 횟수를 매우 보수적으로
  return { retry: true, reason: '429-unknown' };
}

function computeBackoffMs(attempt: number, baseMs: number, capMs: number): number {
  const exp = Math.min(capMs, baseMs * Math.pow(2, attempt));
  // Full jitter: 0..exp
  return Math.floor(Math.random() * exp);
}

async function sleep(ms: number) {
  await new Promise((r) => setTimeout(r, ms));
}

async function withRetry<T>(fn: () => Promise<T>, opts: {
  maxAttempts: number;
  baseMs: number;
  capMs: number;
  deadlineMs: number;
  classify: (e: any) => RetryDecision;
}): Promise<T> {
  const started = Date.now();
  let lastErr: any;

  for (let attempt = 0; attempt < opts.maxAttempts; attempt++) {
    try {
      return await fn();
    } catch (e: any) {
      lastErr = e;
      const decision = opts.classify(e);
      if (!decision.retry) throw e;

      const elapsed = Date.now() - started;
      const remaining = opts.deadlineMs - elapsed;
      if (remaining <= 0) throw e;

      const waitMs = typeof decision.retryAfterMs === 'number'
        ? decision.retryAfterMs
        : computeBackoffMs(attempt, opts.baseMs, opts.capMs);

      // 남은 시간보다 오래 기다릴 수는 없음
      const boundedWait = Math.min(waitMs, Math.max(0, remaining - 10));
      if (boundedWait <= 0) throw e;

      await sleep(boundedWait);
    }
  }

  throw lastErr;
}

function extractHttpStatus(e: any): number | undefined {
  return e?.status ?? e?.response?.status ?? e?.cause?.status;
}

function extractMessage(e: any): string {
  return String(e?.message ?? e?.response?.data?.error?.message ?? e?.toString?.() ?? '');
}

function extractRetryAfterMs(e: any): number | undefined {
  const h = e?.response?.headers;
  if (!h) return;

  // fetch Headers
  if (typeof h.get === 'function') {
    return parseRetryAfterMs(h as Headers);
  }

  // axios style plain object
  const v = h['retry-after'] ?? h['Retry-After'];
  const sec = Number(v);
  if (!Number.isFinite(sec)) return;
  return Math.max(0, sec * 1000);
}

function classifyGeminiError(e: any): RetryDecision {
  const status = extractHttpStatus(e) ?? 0;
  const message = extractMessage(e);
  const retryAfterMs = extractRetryAfterMs(e);

  return decideRetry429({ status, message, retryAfterMs });
}

async function callGeminiWithRetry<T>(call: () => Promise<T>) {
  return withRetry(call, {
    maxAttempts: 5,
    baseMs: 200,
    capMs: 5_000,
    deadlineMs: 12_000,
    classify: classifyGeminiError,
  });
}

class Semaphore {
  private available: number;
  private queue: Array<() => void> = [];

  constructor(max: number) {
    this.available = max;
  }

  async acquire(): Promise<() => void> {
    if (this.available > 0) {
      this.available -= 1;
      return () => this.release();
    }

    await new Promise<void>((resolve) => this.queue.push(resolve));
    this.available -= 1;
    return () => this.release();
  }

  private release() {
    this.available += 1;
    const next = this.queue.shift();
    if (next) next();
  }
}

const geminiSemaphore = new Semaphore(8);

async function guardedGeminiCall<T>(fn: () => Promise<T>): Promise<T> {
  const release = await geminiSemaphore.acquire();
  try {
    return await fn();
  } finally {
    release();
  }
}

type CircuitState = 'closed' | 'open' | 'half-open';

class CircuitBreaker {
  private state: CircuitState = 'closed';
  private openedAt = 0;
  private failures = 0;

  constructor(
    private readonly failureThreshold: number,
    private readonly openMs: number
  ) {}

  canRequest(): boolean {
    if (this.state === 'closed') return true;

    const now = Date.now();
    if (this.state === 'open' && now - this.openedAt >= this.openMs) {
      this.state = 'half-open';
      return true;
    }

    return this.state === 'half-open';
  }

  onSuccess() {
    this.failures = 0;
    this.state = 'closed';
  }

  onFailure() {
    this.failures += 1;
    if (this.failures >= this.failureThreshold) {
      this.state = 'open';
      this.openedAt = Date.now();
    }
  }
}

const geminiCircuit = new CircuitBreaker(10, 30_000);

async function callWithCircuit<T>(fn: () => Promise<T>): Promise<T> {
  if (!geminiCircuit.canRequest()) {
    throw new Error('Gemini circuit is open');
  }

  try {
    const res = await fn();
    geminiCircuit.onSuccess();
    return res;
  } catch (e) {
    geminiCircuit.onFailure();
    throw e;
  }
}