Python UnicodeDecodeError - CSV 인코딩 자동탐지

from pathlib import Path

path = Path("data.csv")
raw = path.read_bytes()[:4]
print(raw)

# UTF-8 BOM: b'\xef\xbb\xbf'
# UTF-16 LE BOM: b'\xff\xfe'
# UTF-16 BE BOM: b'\xfe\xff'

pip install charset-normalizer pandas

from charset_normalizer import from_bytes
import pandas as pd


def detect_encoding(path: str, sample_size: int = 200_000) -> str:
    with open(path, "rb") as f:
        raw = f.read(sample_size)

    # BOM 우선 처리
    if raw.startswith(b"\xef\xbb\xbf"):
        return "utf-8-sig"
    if raw.startswith(b"\xff\xfe") or raw.startswith(b"\xfe\xff"):
        return "utf-16"

    result = from_bytes(raw).best()
    if result is None:
        return "utf-8"  # 보수적 기본값

    return result.encoding


path = "data.csv"
enc = detect_encoding(path)
print("detected:", enc)

# pandas 로딩
# dtype=str: 숫자/날짜 자동추론으로 인한 깨짐을 줄이기 위해 문자열로 먼저 받는 것을 권장
# keep_default_na=False: "NA" 같은 문자열이 NaN으로 바뀌는 것을 방지(필요 시)
df = pd.read_csv(path, encoding=enc, dtype=str, keep_default_na=False)
print(df.head())

import csv
from collections import Counter


def sniff_encoding_and_delimiter(path: str, encodings=None, sample_lines: int = 50):
    if encodings is None:
        encodings = [
            "utf-8-sig",
            "utf-8",
            "cp949",
            "euc-kr",
            "latin1",
        ]

    delimiters = [",", ";", "\t", "|"]

    for enc in encodings:
        try:
            with open(path, "r", encoding=enc, newline="") as f:
                lines = []
                for _ in range(sample_lines):
                    line = f.readline()
                    if not line:
                        break
                    lines.append(line)

            if not lines:
                continue

            # 구분자별로 컬럼 수 안정성 평가
            best = None
            for d in delimiters:
                counts = []
                for line in lines:
                    row = next(csv.reader([line], delimiter=d))
                    counts.append(len(row))

                common, freq = Counter(counts).most_common(1)[0]
                score = freq  # 가장 흔한 컬럼 수가 많이 나오면 안정적

                if best is None or score > best[0]:
                    best = (score, enc, d, common)

            if best and best[0] >= max(3, int(sample_lines * 0.6)):
                _, enc2, delim2, cols = best
                return enc2, delim2, cols

        except UnicodeDecodeError:
            continue

    return None, None, None


path = "data.csv"
enc, delim, cols = sniff_encoding_and_delimiter(path)
print("encoding:", enc, "delimiter:", repr(delim), "cols:", cols)

path = "data.csv"
enc = "cp949"  # 예시

with open(path, "r", encoding=enc, errors="replace") as f:
    text = f.read(1000)
    print(text)

import pandas as pd
import io

path = "data.csv"

raw = open(path, "rb").read()

# 우선 탐지된 인코딩으로 디코딩하되, 깨진 바이트는 치환
text = raw.decode("utf-8", errors="replace")

df = pd.read_csv(io.StringIO(text), dtype=str, keep_default_na=False)
print(df.head())

import pandas as pd

path = "big.csv"
enc = "utf-8-sig"  # detect_encoding 결과라고 가정

chunks = pd.read_csv(
    path,
    encoding=enc,
    dtype=str,
    chunksize=50_000,
    keep_default_na=False,
)

for i, chunk in enumerate(chunks, start=1):
    # 예: 컬럼 정리, 결측 처리, DB 적재 등
    print(i, len(chunk))

from charset_normalizer import from_bytes
import pandas as pd


def detect_encoding_robust(path: str, sample_size: int = 200_000) -> str:
    raw = open(path, "rb").read(sample_size)

    if raw.startswith(b"\xef\xbb\xbf"):
        return "utf-8-sig"
    if raw.startswith(b"\xff\xfe") or raw.startswith(b"\xfe\xff"):
        return "utf-16"

    guess = from_bytes(raw).best()
    enc = guess.encoding if guess else "utf-8"

    # 한국어 CSV 실무 폴백 우선순위
    candidates = [enc, "utf-8-sig", "utf-8", "cp949", "euc-kr"]
    seen = set()
    candidates = [c for c in candidates if not (c in seen or seen.add(c))]

    for c in candidates:
        try:
            # 아주 적은 양을 실제로 디코딩해보며 검증
            raw.decode(c)
            return c
        except UnicodeDecodeError:
            continue

    return "utf-8"  # 최후


def read_csv_safely(path: str) -> pd.DataFrame:
    enc = detect_encoding_robust(path)
    try:
        return pd.read_csv(path, encoding=enc, dtype=str, keep_default_na=False)
    except UnicodeDecodeError:
        # 파일 일부가 깨졌을 가능성: 치환 디코딩 후 재시도
        raw = open(path, "rb").read()
        text = raw.decode(enc, errors="replace")
        import io
        return pd.read_csv(io.StringIO(text), dtype=str, keep_default_na=False)


df = read_csv_safely("data.csv")
print(df.shape)