survival_rate.py

# -*- coding: utf-8 -*-
"""初稿生存率の計測（diff-learning スキル用）

使い方:
    python -X utf8 scripts/survival_rate.py <初稿パス> <最終版パス> [--json] [--th-keep 0.8] [--th-edit 0.5]

初稿を文単位（。！？改行区切り・10字未満除外）に割り、最終版の文と difflib で照合して
文字数加重の 生存(>=th-keep)/改変(th-edit..th-keep)/脱落(<th-edit) を出す。
"""
import argparse
import difflib
import io
import json
import re
import sys
from pathlib import Path

sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding="utf-8", errors="replace")

MIN_SENT_LEN = 10


def load_text(path: str) -> str:
    t = Path(path).read_text(encoding="utf-8", errors="replace")
    # frontmatter
    if t.startswith("---"):
        m = re.match(r"^---\r?\n.*?\r?\n---\r?\n", t, re.S)
        if m:
            t = t[m.end():]
    # HTMLタグ・テンプレタグ・囲み記法の器を落とす（本文テキストだけ比較する）
    t = re.sub(r"<[^>]+>", " ", t)
    t = re.sub(r"\{\{[^}]*\}\}", " ", t)
    t = re.sub(r"^:{2,3}\S*\s*$", " ", t, flags=re.M)
    t = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", t)  # リンクは文言だけ残す
    t = re.sub(r"^#{1,6}\s*", "", t, flags=re.M)
    t = t.replace("==", " ").replace("**", "")
    t = re.sub(r"^\s*\|.*\|\s*$", " ", t, flags=re.M)  # テーブル行は文比較から除外
    t = re.sub(r"^:{2,3}\w+\s*", "", t, flags=re.M)     # 行頭の囲み記法ラベル
    return t


def sentences(text: str):
    parts = re.split(r"(?<=[。！？!?])|\r?\n", text)
    out = []
    for p in parts:
        s = re.sub(r"\s+", " ", (p or "")).strip(" ・-—–―`>*")
        if len(s) >= MIN_SENT_LEN:
            out.append(s)
    return out


def best_match(sent: str, finals, cutoff: float):
    """finals の中で最も似た文と類似度を返す（quick_ratio で足切りして高速化）"""
    sm = difflib.SequenceMatcher()
    sm.set_seq2(sent)
    best, best_r = None, 0.0
    for f in finals:
        sm.set_seq1(f)
        if sm.real_quick_ratio() <= best_r or sm.quick_ratio() <= best_r:
            continue
        r = sm.ratio()
        if r > best_r:
            best, best_r = f, r
            if best_r > 0.99:
                break
    return best, best_r


def main():
    ap = argparse.ArgumentParser(description="初稿と最終版を文単位で照合し、初稿生存率を出す")
    ap.add_argument("initial", help="初稿ファイル")
    ap.add_argument("final", help="最終版ファイル")
    ap.add_argument("--json", action="store_true", help="JSONで出力する")
    ap.add_argument("--th-keep", type=float, default=0.8, help="生存とみなす類似度の下限")
    ap.add_argument("--th-edit", type=float, default=0.5, help="改変とみなす類似度の下限（未満は脱落）")
    ap.add_argument("--max-examples", type=int, default=25, help="実例の最大表示件数")
    a = ap.parse_args()

    init_sents = sentences(load_text(a.initial))
    fin_sents = sentences(load_text(a.final))
    if not init_sents or not fin_sents:
        print("エラー: 文が抽出できませんでした（ファイル・形式を確認）", file=sys.stderr)
        sys.exit(1)

    kept, edited, dropped = [], [], []
    for s in init_sents:
        m, r = best_match(s, fin_sents, a.th_edit)
        if r >= a.th_keep:
            kept.append((s, m, r))
        elif r >= a.th_edit:
            edited.append((s, m, r))
        else:
            dropped.append((s, m, r))

    total = sum(len(s) for s in init_sents) or 1
    w = lambda rows: round(100.0 * sum(len(s) for s, _, _ in rows) / total, 1)
    result = {
        "initial": a.initial,
        "final": a.final,
        "sentences_initial": len(init_sents),
        "sentences_final": len(fin_sents),
        "survival_pct": w(kept),
        "edited_pct": w(edited),
        "dropped_pct": w(dropped),
        "counts": {"kept": len(kept), "edited": len(edited), "dropped": len(dropped)},
    }

    if a.json:
        result["edited_pairs"] = [
            {"initial": s, "final": m, "ratio": round(r, 2)}
            for s, m, r in sorted(edited, key=lambda x: -len(x[0]))[: a.max_examples]
        ]
        result["dropped"] = [s for s, _, _ in sorted(dropped, key=lambda x: -len(x[0]))[: a.max_examples]]
        print(json.dumps(result, ensure_ascii=False, indent=2))
        return

    print("# 初稿生存率レポート")
    print(f"- 初稿: {a.initial}（{len(init_sents)}文） / 最終版: {a.final}（{len(fin_sents)}文）")
    print(f"- **生存 {result['survival_pct']}% ／ 改変 {result['edited_pct']}% ／ 脱落 {result['dropped_pct']}%**（文字数加重）")
    print()
    print(f"## 改変ペア（初稿 → 最終・上位{min(len(edited), a.max_examples)}件）")
    for s, m, r in sorted(edited, key=lambda x: -len(x[0]))[: a.max_examples]:
        print(f"- ({r:.2f}) ❌ {s}")
        print(f"        ⭕ {m}")
    print()
    print(f"## 脱落文（上位{min(len(dropped), a.max_examples)}件）")
    for s, _, _ in sorted(dropped, key=lambda x: -len(x[0]))[: a.max_examples]:
        print(f"- {s}")


if __name__ == "__main__":
    main()