Python UnicodeDecodeError - CSV 인코딩 자동탐지·복구

from pathlib import Path

path = Path("data/report.csv")
raw = path.read_bytes()
print(raw[:200])  # 앞부분 바이트 확인

# pip install charset-normalizer
from charset_normalizer import from_bytes

def detect_encoding(raw: bytes):
    result = from_bytes(raw).best()
    if result is None:
        return None, 0.0
    return result.encoding, float(result.fingerprint.get("confidence", 0.0))

raw = open("data/report.csv", "rb").read()
enc, conf = detect_encoding(raw)
print(enc, conf)

import csv
from pathlib import Path
from charset_normalizer import from_bytes

COMMON_ENCODINGS = [
    "utf-8-sig",  # BOM 포함 UTF-8
    "utf-8",
    "cp949",
    "euc-kr",
    "utf-16",     # 엑셀/윈도우에서 가끔 등장
]

def guess_candidates(raw: bytes):
    # BOM 기반 힌트
    if raw.startswith(b"\xef\xbb\xbf"):
        yield "utf-8-sig"
    if raw.startswith(b"\xff\xfe") or raw.startswith(b"\xfe\xff"):
        yield "utf-16"

    best = from_bytes(raw).best()
    if best and best.encoding:
        yield best.encoding

    for enc in COMMON_ENCODINGS:
        yield enc

    # 마지막 수단: 어떤 바이트든 매핑되는 latin-1
    yield "latin-1"


def read_csv_with_fallback(path: str, delimiter=","):
    p = Path(path)
    raw = p.read_bytes()

    tried = []
    for enc in dict.fromkeys(guess_candidates(raw)):  # 중복 제거 + 순서 유지
        try:
            text = raw.decode(enc)
            # 간단 검증: csv 파싱이 가능한지
            rows = list(csv.reader(text.splitlines(), delimiter=delimiter))
            return enc, rows
        except UnicodeDecodeError:
            tried.append(enc)
        except csv.Error:
            # 디코딩은 됐지만 CSV 구조가 깨진 경우도 있음
            tried.append(f"{enc}(csv-error)")

    raise RuntimeError(f"Failed to read CSV. Tried: {tried}")


enc, rows = read_csv_with_fallback("data/report.csv")
print("encoding=", enc)
print("rows=", len(rows))

import pandas as pd
from pathlib import Path
from charset_normalizer import from_bytes

path = Path("data/report.csv")
raw = path.read_bytes()
enc = (from_bytes(raw).best().encoding if from_bytes(raw).best() else "utf-8")

df = pd.read_csv(path, encoding=enc)

import pandas as pd

df = pd.read_csv(
    "data/report.csv",
    encoding="utf-8",
    encoding_errors="replace",  # 깨진 문자를 U+FFFD로 대체
)

from pathlib import Path
from charset_normalizer import from_bytes

def normalize_to_utf8(src: str, dst: str | None = None):
    src_p = Path(src)
    raw = src_p.read_bytes()

    best = from_bytes(raw).best()
    enc = best.encoding if best and best.encoding else "utf-8"

    text = raw.decode(enc, errors="replace")

    dst_p = Path(dst) if dst else src_p.with_suffix(src_p.suffix + ".utf8.csv")
    dst_p.write_text(text, encoding="utf-8", newline="\n")

    return enc, str(dst_p)


enc, out_path = normalize_to_utf8("data/report.csv")
print("detected=", enc)
print("normalized=", out_path)

from pathlib import Path

def decode_lines_with_report(path: str, encoding: str):
    raw = Path(path).read_bytes()
    lines = raw.splitlines(keepends=False)

    ok = []
    bad = []

    for i, bline in enumerate(lines, start=1):
        try:
            ok.append(bline.decode(encoding))
        except UnicodeDecodeError as e:
            bad.append((i, str(e), bline[:80]))
            ok.append(bline.decode(encoding, errors="replace"))

    return ok, bad


decoded, bad_lines = decode_lines_with_report("data/report.csv", "cp949")
print("bad_lines=", len(bad_lines))
for i, msg, head in bad_lines[:5]:
    print(i, msg, head)

raw = open("data/report.csv", "rb").read()
text = raw.decode("utf-8", errors="replace")
print(text.splitlines()[0])