Python UnicodeDecodeError 해결 - 인코딩 탐지·일괄변환

UnicodeDecodeError: 'utf-8' codec can't decode byte 0xb0 in position 10: invalid start byte

from pathlib import Path

path = Path("data.txt")

# UTF-8 텍스트가 확실할 때
text = path.read_text(encoding="utf-8")

# BOM 가능성이 있을 때(특히 CSV)
text = path.read_text(encoding="utf-8-sig")

text = path.read_text(encoding="cp949")

pip install charset-normalizer

from charset_normalizer import from_bytes
from pathlib import Path

def detect_encoding(path: Path, sample_size: int = 200_000) -> str:
    raw = path.read_bytes()[:sample_size]
    result = from_bytes(raw).best()
    if result is None:
        return "utf-8"  # fallback
    return result.encoding

p = Path("unknown.txt")
enc = detect_encoding(p)
print(enc)
print(p.read_text(encoding=enc, errors="strict"))

from pathlib import Path

CANDIDATES = ["utf-8", "utf-8-sig", "cp949", "euc-kr"]

def read_text_smart(path: Path) -> tuple[str, str]:
    raw = path.read_bytes()

    for enc in CANDIDATES:
        try:
            return raw.decode(enc), enc
        except UnicodeDecodeError:
            continue

    # 최후: 손실 허용(데이터 품질 정책에 따라 선택)
    return raw.decode("utf-8", errors="replace"), "utf-8 (replace)"

text, used = read_text_smart(Path("unknown.txt"))
print("used:", used)

from pathlib import Path

path = Path("unknown.txt")
raw = path.read_bytes()

try:
    raw.decode("utf-8")
except UnicodeDecodeError as e:
    pos = e.start
    window = raw[max(0, pos-16):pos+16]
    print("error:", e)
    print("pos:", pos)
    print("bytes:", window)
    print("hex:", window.hex(" "))

from __future__ import annotations

from dataclasses import dataclass
from pathlib import Path
from typing import Iterable

from charset_normalizer import from_bytes

TEXT_EXTS = {".txt", ".csv", ".tsv", ".log", ".md", ".json"}

@dataclass
class ConvertResult:
    path: Path
    source_encoding: str
    converted: bool
    reason: str = ""


def guess_encoding(raw: bytes) -> str:
    # 1) BOM 빠른 처리
    if raw.startswith(b"\xef\xbb\xbf"):
        return "utf-8-sig"
    if raw.startswith(b"\xff\xfe"):
        return "utf-16"
    if raw.startswith(b"\xfe\xff"):
        return "utf-16"

    # 2) UTF-8 먼저 엄격 시도
    try:
        raw.decode("utf-8")
        return "utf-8"
    except UnicodeDecodeError:
        pass

    # 3) 한국어 환경 후보군
    for enc in ("cp949", "euc-kr"):
        try:
            raw.decode(enc)
            return enc
        except UnicodeDecodeError:
            continue

    # 4) 라이브러리 추정
    best = from_bytes(raw).best()
    return best.encoding if best else "utf-8"


def iter_files(root: Path, exts: set[str] = TEXT_EXTS) -> Iterable[Path]:
    for p in root.rglob("*"):
        if p.is_file() and p.suffix.lower() in exts:
            yield p


def convert_to_utf8(
    path: Path,
    *,
    backup: bool = True,
    newline: str | None = None,
) -> ConvertResult:
    raw = path.read_bytes()

    # 바이너리 간단 필터: 널 바이트가 많으면 텍스트가 아닐 가능성
    if raw.count(b"\x00") > 0:
        return ConvertResult(path, "unknown", False, "contains NUL (maybe binary/utf-16)")

    src_enc = guess_encoding(raw)

    try:
        text = raw.decode(src_enc, errors="strict")
    except UnicodeDecodeError as e:
        return ConvertResult(path, src_enc, False, f"decode failed: {e}")

    # 이미 UTF-8이면 스킵(정책에 따라 재저장할 수도 있음)
    if src_enc in ("utf-8", "utf-8-sig"):
        return ConvertResult(path, src_enc, False, "already utf-8")

    if backup:
        bak = path.with_suffix(path.suffix + ".bak")
        if not bak.exists():
            bak.write_bytes(raw)

    # newline 제어: CSV/로그에서 CRLF/LF 통일이 필요하면 사용
    if newline is not None:
        text = text.replace("\r\n", "\n").replace("\r", "\n")
        if newline != "\n":
            text = text.replace("\n", newline)

    path.write_text(text, encoding="utf-8", newline="")
    return ConvertResult(path, src_enc, True, "converted")


def batch_convert(root: str) -> list[ConvertResult]:
    root_path = Path(root)
    results: list[ConvertResult] = []
    for f in iter_files(root_path):
        results.append(convert_to_utf8(f, backup=True))
    return results


if __name__ == "__main__":
    results = batch_convert("./data")
    converted = [r for r in results if r.converted]
    failed = [r for r in results if (not r.converted and "failed" in r.reason)]

    print("total:", len(results))
    print("converted:", len(converted))
    print("failed:", len(failed))

    for r in failed[:20]:
        print("FAIL", r.path, r.source_encoding, r.reason)

import pandas as pd

# CP949 CSV
df = pd.read_csv("korean.csv", encoding="cp949")

# UTF-8 BOM 포함 CSV(엑셀에서 저장된 파일에 흔함)
df = pd.read_csv("excel.csv", encoding="utf-8-sig")

raw = Path("unknown.txt").read_bytes()
text = raw.decode("utf-8", errors="replace")