Go gRPC DEADLINE_EXCEEDED 원인과 재시도·타임아웃 설계

// 상위 HTTP 요청 ctx에서 내부 호출 예산을 잘라 쓰는 예시
func (s *Server) Handle(ctx context.Context, req *Request) (*Response, error) {
    // 상위 요청이 800ms라면 내부 호출은 250ms만 사용
    childCtx, cancel := context.WithTimeout(ctx, 250*time.Millisecond)
    defer cancel()

    out, err := s.grpcClient.DoSomething(childCtx, &pb.DoReq{Id: req.Id})
    if err != nil {
        return nil, err
    }
    return &Response{Value: out.Value}, nil
}

func (s *Svc) GetUser(ctx context.Context, req *pb.GetUserRequest) (*pb.GetUserResponse, error) {
    // 예: DB 쿼리에 ctx를 전달
    user, err := s.repo.FindUser(ctx, req.Id)
    if err != nil {
        // ctx 만료인지 구분
        if errors.Is(ctx.Err(), context.DeadlineExceeded) {
            return nil, status.Error(codes.DeadlineExceeded, "deadline exceeded")
        }
        return nil, status.Error(codes.Internal, err.Error())
    }

    select {
    case <-ctx.Done():
        // 클라이언트가 이미 취소했다면 즉시 종료
        return nil, status.Error(codes.Canceled, "request canceled")
    default:
    }

    return &pb.GetUserResponse{Id: user.ID, Name: user.Name}, nil
}

func withAttemptTimeout(ctx context.Context, perAttempt time.Duration) (context.Context, context.CancelFunc, error) {
    deadline, ok := ctx.Deadline()
    if !ok {
        // 상위 deadline이 없다면 perAttempt를 그대로 사용
        return context.WithTimeout(ctx, perAttempt)
    }

    remaining := time.Until(deadline)
    if remaining <= 0 {
        return nil, nil, context.DeadlineExceeded
    }

    if remaining < perAttempt {
        perAttempt = remaining
    }
    return context.WithTimeout(ctx, perAttempt)
}

package retry

import (
    "context"
    "math/rand"
    "time"

    "google.golang.org/grpc"
    "google.golang.org/grpc/codes"
    "google.golang.org/grpc/status"
)

type Policy struct {
    MaxAttempts    int
    PerAttemptTO   time.Duration
    BaseBackoff    time.Duration
    MaxBackoff     time.Duration
}

func UnaryClientInterceptor(p Policy) grpc.UnaryClientInterceptor {
    if p.MaxAttempts <= 0 {
        p.MaxAttempts = 1
    }
    if p.BaseBackoff <= 0 {
        p.BaseBackoff = 20 * time.Millisecond
    }
    if p.MaxBackoff <= 0 {
        p.MaxBackoff = 200 * time.Millisecond
    }

    return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn,
        invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error {

        var lastErr error

        for attempt := 1; attempt <= p.MaxAttempts; attempt++ {
            attemptCtx, cancel, err := func() (context.Context, context.CancelFunc, error) {
                if p.PerAttemptTO <= 0 {
                    // per-attempt timeout을 강제하지 않음
                    return ctx, func() {}, nil
                }
                c, cancel := context.WithTimeout(ctx, p.PerAttemptTO)
                // 상위 ctx가 이미 만료되면 여기서도 빠르게 실패
                if err := ctx.Err(); err != nil {
                    cancel()
                    return nil, nil, err
                }
                return c, cancel, nil
            }()
            if err != nil {
                return err
            }

            lastErr = invoker(attemptCtx, method, req, reply, cc, opts...)
            cancel()

            if lastErr == nil {
                return nil
            }

            st, ok := status.FromError(lastErr)
            if !ok {
                // gRPC status가 아니라면 재시도하지 않음
                return lastErr
            }

            if !isRetryable(st.Code()) {
                return lastErr
            }

            // 마지막 attempt면 종료
            if attempt == p.MaxAttempts {
                break
            }

            // 상위 deadline이 남아있는지 확인
            if err := ctx.Err(); err != nil {
                return err
            }

            // backoff with jitter
            backoff := expBackoff(p.BaseBackoff, p.MaxBackoff, attempt)
            jitter := time.Duration(rand.Int63n(int64(backoff / 5))) // 0~20%
            sleep := backoff + jitter

            t := time.NewTimer(sleep)
            select {
            case <-ctx.Done():
                t.Stop()
                return ctx.Err()
            case <-t.C:
            }
        }

        return lastErr
    }
}

func isRetryable(c codes.Code) bool {
    switch c {
    case codes.Unavailable, codes.DeadlineExceeded:
        return true
    default:
        return false
    }
}

func expBackoff(base, max time.Duration, attempt int) time.Duration {
    // attempt=1이면 1배, 2면 2배, 3이면 4배...
    b := base << (attempt - 1)
    if b > max {
        return max
    }
    return b
}

conn, err := grpc.NewClient(
    target,
    grpc.WithTransportCredentials(creds),
    grpc.WithChainUnaryInterceptor(
        retry.UnaryClientInterceptor(retry.Policy{
            MaxAttempts:  3,
            PerAttemptTO: 150 * time.Millisecond,
            BaseBackoff:  30 * time.Millisecond,
            MaxBackoff:   200 * time.Millisecond,
        }),
    ),
)
if err != nil {
    panic(err)
}