#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
cq2_log_analyzer.py — 超群学道网 nginx 日志深度分析器
======================================================
适配日志格式 xq_main:
  $host $remote_addr "$http_x_forwarded_for" [$time_local]
  "$request" $status $body_bytes_sent
  "$http_referer" "$http_user_agent"
  rt=$request_time cache=$upstream_cache_status

功能：
  1. 流量分类：搜索蜘蛛 / AI蜘蛛 / SEO工具 / 社交预览 / 脚本工具 / 未知Bot / 真人
  2. 蜘蛛验真：对声称 Baiduspider/Googlebot/bingbot 等的高频 IP 做 rDNS 反查 + 正向确认
  3. 搜索蜘蛛专项：百度/Google/Bing 抓取量、独立URL数、新旧内容占比(/website/)、状态码
  4. 子域名(host)维度、内容板块维度、状态码、缓存命中率、响应时间分布
  5. 真人流量：粗略UV/PV、搜索引擎来路点击(自然流量!)、Top落地页
  6. 蜘蛛陷阱(/trap/)命中者名单
  7. 可选 CSV / JSON 导出，支持 .gz 轮转日志与多文件

用法示例：
  python3 cq2_log_analyzer.py /www/wwwlogs/cq2.cn.log
  python3 cq2_log_analyzer.py /www/wwwlogs/cq2.cn.log --day 2026-07-12
  python3 cq2_log_analyzer.py /www/wwwlogs/cq2.cn*.log --day yesterday --rdns-top 100
  python3 cq2_log_analyzer.py /www/wwwlogs/cq2.cn.log --no-rdns --csv-dir /root/logrpt
  python3 cq2_log_analyzer.py /www/wwwlogs/cq2.cn.log --host jm.cq2.cn --day today

依赖：Python 3.9+ 标准库，无需 pip。
"""

import argparse
import concurrent.futures
import glob
import gzip
import ipaddress
import json
import os
import re
import socket
import sys
from collections import Counter, defaultdict
from datetime import datetime, timedelta

# ----------------------------------------------------------------------------
# 日志行解析
# ----------------------------------------------------------------------------
LINE_RE = re.compile(
    r'^(?P<host>\S+)\s+(?P<ip>\S+)\s+"(?P<xff>[^"]*)"\s+\[(?P<time>[^\]]+)\]\s+'
    r'"(?P<request>[^"]*)"\s+(?P<status>\d{3})\s+(?P<bytes>\d+|-)\s+'
    r'"(?P<referer>[^"]*)"\s+"(?P<ua>[^"]*)"\s+'
    r'rt=(?P<rt>[\d.]+|-)\s+cache=(?P<cache>\S+)'
)

MONTHS = {'Jan': '01', 'Feb': '02', 'Mar': '03', 'Apr': '04', 'May': '05', 'Jun': '06',
          'Jul': '07', 'Aug': '08', 'Sep': '09', 'Oct': '10', 'Nov': '11', 'Dec': '12'}

STATIC_EXT = {'.css', '.js', '.png', '.jpg', '.jpeg', '.gif', '.webp', '.svg', '.ico',
              '.woff', '.woff2', '.ttf', '.mp3', '.mp4', '.map', '.json', '.xml', '.txt'}

# ----------------------------------------------------------------------------
# Bot 分类规则（顺序即优先级；name 用于聚合展示）
# 类别: SEARCH 搜索蜘蛛 | AI AI蜘蛛 | SEO SEO工具 | SOCIAL 社交预览
#       MONITOR 监控 | TOOL 脚本/工具 | BOT 其他Bot
# ----------------------------------------------------------------------------
BOT_RULES = [
    # --- 搜索引擎蜘蛛 ---
    ('Baiduspider',        r'Baiduspider(?!-render)', 'SEARCH'),
    ('Baiduspider-render', r'Baiduspider-render',     'SEARCH'),
    ('Googlebot',          r'Googlebot(?!-Image)',    'SEARCH'),
    ('Googlebot-Image',    r'Googlebot-Image',        'SEARCH'),
    ('bingbot',            r'bingbot',                'SEARCH'),
    ('Sogou Spider',       r'Sogou\s?(web|inst|Pic)? ?spider', 'SEARCH'),
    ('360Spider',          r'360Spider|HaoSouSpider', 'SEARCH'),
    ('YisouSpider(神马)',   r'YisouSpider',            'SEARCH'),
    ('PetalBot',           r'PetalBot',               'SEARCH'),
    ('YandexBot',          r'YandexBot|YandexImages', 'SEARCH'),
    ('DuckDuckBot',        r'DuckDuckBot|DuckDuckGo', 'SEARCH'),
    ('Applebot',           r'Applebot(?!-Extended)',  'SEARCH'),
    ('Yeti(Naver)',        r'\bYeti\b',               'SEARCH'),
    ('SeznamBot',          r'SeznamBot',              'SEARCH'),
    ('coccocbot',          r'coccocbot',              'SEARCH'),

    # --- AI 蜘蛛 / AI 应用取回 ---
    ('GPTBot',             r'GPTBot',                 'AI'),
    ('OAI-SearchBot',      r'OAI-SearchBot',          'AI'),
    ('ChatGPT-User',       r'ChatGPT-User',           'AI'),
    ('ClaudeBot',          r'ClaudeBot|claudebot',    'AI'),
    ('Claude-User/Search', r'Claude-(User|SearchBot|Web)|anthropic-ai', 'AI'),
    ('PerplexityBot',      r'PerplexityBot',          'AI'),
    ('Perplexity-User',    r'Perplexity-User',        'AI'),
    ('Bytespider(字节)',    r'Bytespider',             'AI'),
    ('DouBao(豆包)',        r'DoubaoBot|Doubao',       'AI'),
    ('Amazonbot',          r'Amazonbot',              'AI'),
    ('meta-external',      r'meta-externalagent|meta-externalfetcher|FacebookBot', 'AI'),
    ('Applebot-Extended',  r'Applebot-Extended',      'AI'),
    ('Google-Extended',    r'Google-Extended|GoogleOther', 'AI'),
    ('CCBot',              r'CCBot',                  'AI'),
    ('cohere',             r'cohere-ai|cohere-train', 'AI'),
    ('Diffbot',            r'Diffbot',                'AI'),
    ('Timpibot',           r'Timpibot',               'AI'),
    ('omgili',             r'omgili',                 'AI'),
    ('ImagesiftBot',       r'ImagesiftBot',           'AI'),
    ('YouBot',             r'\bYouBot\b',             'AI'),
    ('MistralAI-User',     r'MistralAI',              'AI'),
    ('DeepSeek',           r'DeepSeek',               'AI'),
    ('KimiBot(月之暗面)',   r'KimiBot|MoonshotBot|Moonshot', 'AI'),
    ('QwenBot(通义)',       r'QwenBot|TongyiBot|Qwen', 'AI'),
    ('PanguBot',           r'PanguBot',               'AI'),
    ('AI2Bot',             r'AI2Bot|allenai',         'AI'),

    # --- SEO / 数据工具 ---
    ('AhrefsBot',          r'AhrefsBot',              'SEO'),
    ('SemrushBot',         r'SemrushBot',             'SEO'),
    ('MJ12bot',            r'MJ12bot',                'SEO'),
    ('DotBot',             r'DotBot',                 'SEO'),
    ('DataForSeoBot',      r'DataForSeoBot',          'SEO'),
    ('BLEXBot',            r'BLEXBot',                'SEO'),
    ('serpstatbot',        r'serpstatbot',            'SEO'),
    ('Barkrowler',         r'Barkrowler',             'SEO'),
    ('ZoominfoBot',        r'ZoominfoBot',            'SEO'),
    ('MegaIndex',          r'MegaIndex',              'SEO'),

    # --- 社交/IM 链接预览 ---
    ('facebookexternalhit', r'facebookexternalhit',   'SOCIAL'),
    ('Twitterbot',         r'Twitterbot',             'SOCIAL'),
    ('TelegramBot',        r'TelegramBot',            'SOCIAL'),
    ('Slackbot',           r'Slackbot',               'SOCIAL'),
    ('Discordbot',         r'Discordbot',             'SOCIAL'),
    ('LinkedInBot',        r'LinkedInBot',            'SOCIAL'),
    ('WhatsApp',           r'WhatsApp',               'SOCIAL'),
    ('QQ/WeChat preview',  r'MicroMessenger.+(Bot|preview)|QQBrowser.+Bot', 'SOCIAL'),

    # --- 监控 ---
    ('UptimeRobot',        r'UptimeRobot',            'MONITOR'),
    ('Pingdom',            r'Pingdom',                'MONITOR'),
    ('StatusCake',         r'StatusCake',             'MONITOR'),

    # --- 脚本 / HTTP 库（多为采集器）---
    ('python-requests',    r'python-requests',        'TOOL'),
    ('Python-urllib',      r'Python-urllib|python-httpx|aiohttp', 'TOOL'),
    ('curl',               r'^curl/|curl/',           'TOOL'),
    ('wget',               r'Wget',                   'TOOL'),
    ('Go-http-client',     r'Go-http-client',         'TOOL'),
    ('okhttp',             r'okhttp',                 'TOOL'),
    ('Java',               r'^Java/|Apache-HttpClient', 'TOOL'),
    ('Scrapy',             r'Scrapy',                 'TOOL'),
    ('node-fetch/axios',   r'node-fetch|axios',       'TOOL'),
    ('empty-UA',           r'^-?$',                   'TOOL'),
]
BOT_RULES = [(n, re.compile(p, re.I), c) for n, p, c in BOT_RULES]
GENERIC_BOT_RE = re.compile(r'bot|spider|crawl|slurp|fetch|scan|monitor|archive|httrack|harvest', re.I)

# 需要 rDNS 验真的蜘蛛及其合法 PTR 后缀（None = 官方无稳定 rDNS，标记为"无法验证"）
VERIFY_SUFFIX = {
    'Baiduspider':        ('.baidu.com', '.baidu.jp'),
    'Baiduspider-render': ('.baidu.com', '.baidu.jp'),
    'Googlebot':          ('.googlebot.com', '.google.com'),
    'Googlebot-Image':    ('.googlebot.com', '.google.com'),
    'bingbot':            ('.search.msn.com',),
    'Sogou Spider':       ('.sogou.com', '.crawl.sogou.com'),
    'YandexBot':          ('.yandex.ru', '.yandex.net', '.yandex.com'),
    'Applebot':           ('.applebot.apple.com',),
    'PetalBot':           ('.petalsearch.com', '.aspiegel.com'),
    'Amazonbot':          ('.crawl.amazonbot.amazon',),
    'Yeti(Naver)':        ('.naver.com',),
    '360Spider':          None,
    'YisouSpider(神马)':   None,
    'GPTBot':             None,
    'ClaudeBot':          None,
    'Bytespider(字节)':    None,
}

SEARCH_REFERER_RE = re.compile(
    r'baidu\.com|google\.[a-z.]+|bing\.com|so\.com|sm\.cn|sogou\.com|'
    r'duckduckgo|yandex|quark|toutiao|yahoo', re.I)


def classify_ua(ua):
    """返回 (类别, 名称)。类别: SEARCH/AI/SEO/SOCIAL/MONITOR/TOOL/BOT/HUMAN"""
    for name, rx, cat in BOT_RULES:
        if rx.search(ua):
            return cat, name
    if GENERIC_BOT_RE.search(ua):
        m = re.search(r'([A-Za-z0-9._-]{2,30}(?:bot|spider|crawler))', ua, re.I)
        return 'BOT', (m.group(1) if m else 'unknown-bot')
    return 'HUMAN', 'human'


def classify_section(host, path):
    """按站点结构归类 URL 板块"""
    p = path.split('?', 1)[0].lower()
    ext = os.path.splitext(p)[1]
    if ext in STATIC_EXT or p.startswith(('/assets/', '/static/')):
        return '静态资源'
    if '/trap/' in p:
        return '⚠蜘蛛陷阱'
    if p == '/' or p == '/index.html':
        return '首页'
    sub = host.split('.')[0]
    if '/website/' in p or '/contribute' in p or '/webstack' in p:
        return '⚠旧导航内容'
    if sub in ('rl', 'wnl') and re.search(r'/\d{4}-\d{2}-\d{2}', p):
        return '黄历日期页'
    if sub == 'jm' and 'mengjian-' in p:
        return '解梦词条'
    if sub in ('zi', 'ci', 'bjx'):
        return '字词/姓氏条目'
    if sub in ('shici', 'dl', 'shu', 'mr'):
        return '诗文典籍'
    if sub in ('bz', 'qm', 'xz', 'sx') :
        return '命理工具'
    if '/search' in p:
        return '站内搜索'
    if '/page/' in p:
        return '关于/条款页'
    if any(k in p for k in ('/wuxing', '/chenggu', '/haoma', '/jintian', '/caimi',
                            '/mingju', '/mjgs', '/shenhua', '/mingyan', '/yishu',
                            '/gdwx', '/jfzh', '/wenzhang')):
        return '主站栏目页'
    return '其他'


def parse_time_day(t):
    # "13/Jul/2026:00:00:01 +0800" -> "2026-07-13"
    return f'{t[7:11]}-{MONTHS.get(t[3:6], "00")}-{t[0:2]}'


def is_public_ip(s):
    try:
        ip = ipaddress.ip_address(s)
        return not (ip.is_private or ip.is_loopback or ip.is_reserved or ip.is_link_local)
    except ValueError:
        return False


# ----------------------------------------------------------------------------
# rDNS 验真
# ----------------------------------------------------------------------------
def rdns_verify(items, cache_path, workers=16, timeout=3.0):
    """items: [(ip, bot_name)] -> {ip: (ptr, verdict)}
       verdict: VERIFIED / FAKE / NO-PTR / UNVERIFIABLE / N-A"""
    cache = {}
    if os.path.exists(cache_path):
        try:
            cache = json.load(open(cache_path))
        except Exception:
            cache = {}

    socket.setdefaulttimeout(timeout)

    def lookup(ip):
        if ip in cache:
            return ip, cache[ip]
        try:
            ptr = socket.gethostbyaddr(ip)[0]
        except Exception:
            ptr = ''
        fwd_ok = False
        if ptr:
            try:
                fwd_ok = ip in {ai[4][0] for ai in socket.getaddrinfo(ptr, None)}
            except Exception:
                fwd_ok = False
        cache[ip] = [ptr, fwd_ok]
        return ip, [ptr, fwd_ok]

    results = {}
    with concurrent.futures.ThreadPoolExecutor(max_workers=workers) as ex:
        for ip, res in ex.map(lookup, {ip for ip, _ in items}):
            results[ip] = res

    try:
        json.dump(cache, open(cache_path, 'w'))
    except Exception:
        pass

    verdicts = {}
    for ip, bot in items:
        ptr, fwd_ok = results.get(ip, ['', False])
        suffixes = VERIFY_SUFFIX.get(bot, 'N-A')
        if suffixes == 'N-A':
            verdicts[ip] = (ptr, 'N-A')          # 非重点验真对象，仅展示 PTR
        elif suffixes is None:
            verdicts[ip] = (ptr, 'UNVERIFIABLE')  # 官方无 rDNS 方案
        elif not ptr:
            verdicts[ip] = ('', 'NO-PTR→疑似伪造')
        elif any(ptr.lower().endswith(s) for s in suffixes) and fwd_ok:
            verdicts[ip] = (ptr, 'VERIFIED✓')
        elif any(ptr.lower().endswith(s) for s in suffixes):
            verdicts[ip] = (ptr, '后缀匹配但正查失败')
        else:
            verdicts[ip] = (ptr, 'FAKE✗伪造')
    return verdicts


# ----------------------------------------------------------------------------
# 主分析器
# ----------------------------------------------------------------------------
class Stats:
    def __init__(self):
        self.total = 0
        self.malformed = 0
        self.days = Counter()
        self.by_cat = Counter()                       # 类别请求数
        self.by_cat_bytes = Counter()
        self.by_cat_ips = defaultdict(set)
        self.by_bot = Counter()                       # bot 名称请求数
        self.bot_ips = defaultdict(Counter)           # bot -> ip -> n
        self.ip_ua_sample = {}
        self.by_host = Counter()
        self.by_host_cat = defaultdict(Counter)
        self.status = Counter()
        self.status_by_cat = defaultdict(Counter)
        self.cache = Counter()
        self.rt_sum = defaultdict(float)
        self.rt_n = Counter()
        self.rt_slow = Counter()                      # rt > 1s
        self.section_by_cat = defaultdict(Counter)
        self.spider_urls = defaultdict(set)           # 重点蜘蛛独立URL
        self.spider_daily = defaultdict(Counter)      # 重点蜘蛛按天
        self.spider_status = defaultdict(Counter)
        self.spider_hosts = defaultdict(Counter)
        self.err404 = Counter()
        self.err5xx = Counter()
        self.trap_hits = Counter()                    # (cat,name) -> n
        self.trap_ips = Counter()
        # 真人
        self.human_ip_day = set()                     # (day, ip) 粗略UV
        self.human_pv = 0
        self.human_landing = Counter()
        self.organic_ref = Counter()
        self.human_ref_pv = 0

    FOCUS_SPIDERS = ('Baiduspider', 'Baiduspider-render', 'Googlebot', 'bingbot',
                     'Sogou Spider', '360Spider', 'YisouSpider(神马)')

    def feed(self, m):
        self.total += 1
        host = m['host']
        ip = m['ip']
        xff = m['xff']
        if xff and xff != '-':
            first = xff.split(',')[0].strip()
            if is_public_ip(first):
                ip = first                            # CDN 场景取真实客户端 IP
        day = parse_time_day(m['time'])
        self.days[day] += 1

        req = m['request']
        parts = req.split(' ')
        path = parts[1] if len(parts) >= 2 else req
        status = m['status']
        nbytes = 0 if m['bytes'] == '-' else int(m['bytes'])
        ua = m['ua']
        cat, name = classify_ua(ua)
        sec = classify_section(host, path)

        self.by_cat[cat] += 1
        self.by_cat_bytes[cat] += nbytes
        self.by_cat_ips[cat].add(ip)
        self.by_host[host] += 1
        self.by_host_cat[host][cat] += 1
        self.status[status] += 1
        self.status_by_cat[cat][status] += 1
        self.cache[m['cache']] += 1
        self.section_by_cat[cat][sec] += 1

        if m['rt'] != '-':
            rt = float(m['rt'])
            self.rt_sum[cat] += rt
            self.rt_n[cat] += 1
            if rt > 1.0:
                self.rt_slow[cat] += 1

        if cat != 'HUMAN':
            self.by_bot[name] += 1
            self.bot_ips[name][ip] += 1
            self.ip_ua_sample.setdefault(ip, ua[:120])
            if name in self.FOCUS_SPIDERS:
                self.spider_daily[name][day] += 1
                self.spider_status[name][status] += 1
                self.spider_hosts[name][host] += 1
                if sec != '静态资源':
                    self.spider_urls[name].add(host + path.split('?')[0])
        else:
            self.human_pv += 1
            self.human_ip_day.add((day, ip))
            ref = m['referer']
            if sec != '静态资源':
                self.human_landing[host + path.split('?')[0]] += 1
                if ref and ref != '-' and SEARCH_REFERER_RE.search(ref):
                    eng = SEARCH_REFERER_RE.search(ref).group(0).lower()
                    self.organic_ref[eng] += 1
                    self.human_ref_pv += 1

        if sec == '⚠蜘蛛陷阱':
            self.trap_hits[f'{cat}:{name}'] += 1
            self.trap_ips[ip] += 1
        if status == '404':
            self.err404[host + path.split('?')[0]] += 1
        elif status.startswith('5'):
            self.err5xx[host + path.split('?')[0]] += 1


def open_any(path):
    return gzip.open(path, 'rt', errors='replace') if path.endswith('.gz') \
        else open(path, 'r', errors='replace')


def fmt_n(n):
    return f'{n:,}'


def bar(pct, width=24):
    full = int(pct / 100 * width)
    return '█' * full + '·' * (width - full)


def table(rows, headers):
    cols = list(range(len(headers)))
    w = [max(len(str(headers[i])), *(len(str(r[i])) for r in rows)) if rows else len(str(headers[i])) for i in cols]
    out = ['  '.join(str(headers[i]).ljust(w[i]) for i in cols),
           '  '.join('-' * w[i] for i in cols)]
    for r in rows:
        out.append('  '.join(str(r[i]).ljust(w[i]) for i in cols))
    return '\n'.join(out)


CAT_LABEL = {'SEARCH': '搜索蜘蛛', 'AI': 'AI蜘蛛', 'SEO': 'SEO工具', 'SOCIAL': '社交预览',
             'MONITOR': '监控', 'TOOL': '脚本/HTTP库', 'BOT': '未知Bot', 'HUMAN': '真人'}


def report(st: Stats, args):
    L = []
    add = L.append
    days = sorted(st.days)
    add('=' * 78)
    add(f'  超群学道网日志分析报告  |  {days[0] if days else "?"} ~ {days[-1] if days else "?"}'
        f'  |  总请求 {fmt_n(st.total)}  (解析失败 {st.malformed})')
    add('=' * 78)

    # 1. 按天
    add('\n【1】每日请求量')
    add(table([(d, fmt_n(n)) for d, n in sorted(st.days.items())], ('日期', '请求数')))

    # 2. 流量构成
    add('\n【2】流量构成（按请求）')
    rows = []
    for cat, n in st.by_cat.most_common():
        pct = n / st.total * 100
        avg_rt = st.rt_sum[cat] / st.rt_n[cat] if st.rt_n[cat] else 0
        rows.append((CAT_LABEL.get(cat, cat), fmt_n(n), f'{pct:5.1f}%', bar(pct),
                     fmt_n(len(st.by_cat_ips[cat])),
                     f'{st.by_cat_bytes[cat] / 1024 / 1024 / 1024:.2f}GB',
                     f'{avg_rt * 1000:.0f}ms'))
    add(table(rows, ('类别', '请求', '占比', '', '独立IP', '流量', '平均rt')))

    # 3. Top Bot
    add(f'\n【3】Top {args.top} Bot（按请求量）')
    rows = [(name, fmt_n(n), f'{n / st.total * 100:4.1f}%', fmt_n(len(st.bot_ips[name])))
            for name, n in st.by_bot.most_common(args.top)]
    add(table(rows, ('Bot', '请求', '占比', '独立IP')))

    # 4. 蜘蛛验真
    if args.verify_results:
        add(f'\n【4】高频蜘蛛 IP 验真（rDNS 反查 + 正向确认，Top {args.rdns_top} IP）')
        rows = []
        for bot, ip, n in args.verify_items_meta:
            ptr, verdict = args.verify_results.get(ip, ('', '?'))
            rows.append((bot[:20], ip, fmt_n(n), (ptr or '(无PTR)')[:44], verdict))
        add(table(rows, ('Bot', 'IP', '请求', 'PTR记录', '判定')))
        fake = sum(n for b, i, n in args.verify_items_meta
                   if args.verify_results.get(i, ('', ''))[1].startswith(('FAKE', 'NO-PTR')))
        add(f'\n  ⚠ 判定为伪造/无PTR的请求合计约 {fmt_n(fake)} 次（仅统计已验真IP），'
            f'建议在 CDN/nginx 层封禁 FAKE IP。')
    else:
        add('\n【4】蜘蛛验真：已跳过（--no-rdns）')

    # 5. 搜索蜘蛛专项
    add('\n【5】搜索蜘蛛专项（冷启动核心指标）')
    for name in Stats.FOCUS_SPIDERS:
        tot = sum(st.spider_daily[name].values())
        if not tot:
            continue
        uniq = len(st.spider_urls[name])
        old = sum(n for s_, n in st.section_by_cat['SEARCH'].items() if s_ == '⚠旧导航内容')
        add(f'\n  ◆ {name}: 总抓取 {fmt_n(tot)} | 独立URL {fmt_n(uniq)} | 重复抓取率 '
            f'{(1 - uniq / tot) * 100 if tot else 0:.0f}%')
        dd = '  '.join(f'{d[5:]}:{fmt_n(n)}' for d, n in sorted(st.spider_daily[name].items()))
        add(f'    按天: {dd}')
        sc = '  '.join(f'{s}:{fmt_n(n)}' for s, n in st.spider_status[name].most_common(6))
        add(f'    状态码: {sc}')
        hh = '  '.join(f'{h}:{fmt_n(n)}' for h, n in st.spider_hosts[name].most_common(8))
        add(f'    子域分布: {hh}')
    add(f'\n  ⚠ 搜索蜘蛛抓取"旧导航内容"合计: '
        f'{fmt_n(st.section_by_cat["SEARCH"].get("⚠旧导航内容", 0))} 次'
        f'（此数字应趋近 0，否则死链处理未生效）')

    # 6. 内容板块 × 类别
    add('\n【6】各类流量抓取的内容板块分布')
    for cat in ('SEARCH', 'AI', 'HUMAN'):
        if not st.section_by_cat[cat]:
            continue
        tot = sum(st.section_by_cat[cat].values())
        seg = '  '.join(f'{s}:{n / tot * 100:.0f}%' for s, n in st.section_by_cat[cat].most_common(8))
        add(f'  {CAT_LABEL[cat]}: {seg}')

    # 7. 子域名
    add(f'\n【7】子域名流量 Top {args.top}')
    rows = []
    for h, n in st.by_host.most_common(args.top):
        c = st.by_host_cat[h]
        rows.append((h, fmt_n(n),
                     fmt_n(c.get('SEARCH', 0)), fmt_n(c.get('AI', 0)), fmt_n(c.get('HUMAN', 0))))
    add(table(rows, ('host', '总请求', '搜索蜘蛛', 'AI蜘蛛', '真人')))

    # 8. 状态码 & 缓存
    add('\n【8】状态码 / 缓存命中')
    add('  状态码: ' + '  '.join(f'{s}:{fmt_n(n)}' for s, n in st.status.most_common(8)))
    tot_c = sum(st.cache.values())
    add('  缓存:   ' + '  '.join(f'{k}:{n / tot_c * 100:.0f}%' for k, n in st.cache.most_common(6)))
    if st.err404:
        add('\n  Top 404:')
        for u, n in st.err404.most_common(8):
            add(f'    {fmt_n(n):>8}  {u[:88]}')
    if st.err5xx:
        add('\n  ⚠ Top 5xx:')
        for u, n in st.err5xx.most_common(8):
            add(f'    {fmt_n(n):>8}  {u[:88]}')

    # 9. 蜘蛛陷阱
    if st.trap_hits:
        add('\n【9】/trap/ 蜘蛛陷阱命中者（不守 robots 的爬虫名单，可直接封禁）')
        add(table([(k, fmt_n(n)) for k, n in st.trap_hits.most_common(15)], ('类别:名称', '次数')))
        add('  Top 触发IP: ' + '  '.join(f'{i}({n})' for i, n in st.trap_ips.most_common(8)))

    # 10. 真人流量
    add('\n【10】真人流量（JS 无关的日志口径，可与第三方统计交叉验证）')
    uv_by_day = Counter(d for d, _ in st.human_ip_day)
    add('  粗略UV(独立IP/天): ' + '  '.join(f'{d[5:]}:{fmt_n(n)}' for d, n in sorted(uv_by_day.items())))
    add(f'  真人PV(非静态): {fmt_n(sum(st.human_landing.values()))} | 全部真人请求: {fmt_n(st.human_pv)}')
    if st.organic_ref:
        add(f'  搜索引擎来路点击(自然流量!): {fmt_n(st.human_ref_pv)} 次 → '
            + '  '.join(f'{e}:{n}' for e, n in st.organic_ref.most_common()))
    else:
        add('  搜索引擎来路点击: 0（自然流量尚未起量）')
    add('  Top 真人落地页:')
    for u, n in st.human_landing.most_common(10):
        add(f'    {fmt_n(n):>8}  {u[:88]}')

    add('\n' + '=' * 78)
    return '\n'.join(L)


def main():
    ap = argparse.ArgumentParser(description='超群学道网 nginx 日志深度分析器')
    ap.add_argument('logs', nargs='+', help='日志文件（支持通配与 .gz）')
    ap.add_argument('--day', help='只分析某天: 2026-07-12 / today / yesterday')
    ap.add_argument('--host', help='只分析某子域名，如 jm.cq2.cn')
    ap.add_argument('--top', type=int, default=20)
    ap.add_argument('--rdns-top', type=int, default=80, help='验真的高频IP数量上限')
    ap.add_argument('--no-rdns', action='store_true', help='跳过 rDNS 验真（离线/快速模式）')
    ap.add_argument('--csv-dir', help='导出 CSV 到目录')
    ap.add_argument('--json-out', help='导出汇总 JSON 到文件')
    args = ap.parse_args()

    day_filter = None
    if args.day:
        if args.day == 'today':
            day_filter = datetime.now().strftime('%Y-%m-%d')
        elif args.day == 'yesterday':
            day_filter = (datetime.now() - timedelta(days=1)).strftime('%Y-%m-%d')
        else:
            day_filter = args.day

    files = []
    for p in args.logs:
        files.extend(sorted(glob.glob(p)) or [p])

    st = Stats()
    for f in files:
        if not os.path.exists(f):
            print(f'!! 找不到文件: {f}', file=sys.stderr)
            continue
        with open_any(f) as fh:
            for line in fh:
                m = LINE_RE.match(line)
                if not m:
                    st.malformed += 1
                    continue
                d = m.groupdict()
                if day_filter and parse_time_day(d['time']) != day_filter:
                    continue
                if args.host and d['host'] != args.host:
                    continue
                st.feed(d)

    if not st.total:
        print('没有匹配的日志行（检查 --day / --host / 日志格式）')
        return

    # rDNS 验真：取每个重点 bot 的高频 IP + 全局高频 bot IP
    args.verify_results = None
    args.verify_items_meta = []
    if not args.no_rdns:
        items = []
        seen = set()
        for bot in list(VERIFY_SUFFIX) + [b for b, _ in st.by_bot.most_common(10)]:
            for ip, n in st.bot_ips.get(bot, Counter()).most_common(8):
                if ip not in seen:
                    seen.add(ip)
                    items.append((ip, bot))
                    args.verify_items_meta.append((bot, ip, n))
                if len(items) >= args.rdns_top:
                    break
            if len(items) >= args.rdns_top:
                break
        cache_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), '.rdns_cache.json')
        print(f'.. 正在对 {len(items)} 个高频IP做 rDNS 验真（首次较慢，结果会缓存）', file=sys.stderr)
        args.verify_results = rdns_verify(items, cache_path)
        args.verify_items_meta.sort(key=lambda x: -x[2])

    print(report(st, args))

    # CSV 导出
    if args.csv_dir:
        os.makedirs(args.csv_dir, exist_ok=True)
        tag = day_filter or 'all'
        with open(os.path.join(args.csv_dir, f'category_{tag}.csv'), 'w') as f:
            f.write('category,requests,unique_ips,bytes\n')
            for cat, n in st.by_cat.most_common():
                f.write(f'{CAT_LABEL.get(cat, cat)},{n},{len(st.by_cat_ips[cat])},{st.by_cat_bytes[cat]}\n')
        with open(os.path.join(args.csv_dir, f'bots_{tag}.csv'), 'w') as f:
            f.write('bot,requests,unique_ips\n')
            for b, n in st.by_bot.most_common():
                f.write(f'{b},{n},{len(st.bot_ips[b])}\n')
        with open(os.path.join(args.csv_dir, f'hosts_{tag}.csv'), 'w') as f:
            f.write('host,total,search,ai,human\n')
            for h, n in st.by_host.most_common():
                c = st.by_host_cat[h]
                f.write(f'{h},{n},{c.get("SEARCH",0)},{c.get("AI",0)},{c.get("HUMAN",0)}\n')
        print(f'.. CSV 已导出到 {args.csv_dir}', file=sys.stderr)

    # JSON 汇总
    if args.json_out:
        js = {
            'range': sorted(st.days),
            'total': st.total,
            'by_category': {CAT_LABEL.get(k, k): v for k, v in st.by_cat.items()},
            'top_bots': st.by_bot.most_common(30),
            'human_uv_by_day': dict(Counter(d for d, _ in st.human_ip_day)),
            'organic_clicks': dict(st.organic_ref),
            'spider_daily': {k: dict(v) for k, v in st.spider_daily.items()},
            'old_content_crawls_by_search': st.section_by_cat['SEARCH'].get('⚠旧导航内容', 0),
        }
        json.dump(js, open(args.json_out, 'w'), ensure_ascii=False, indent=1)
        print(f'.. JSON 已导出到 {args.json_out}', file=sys.stderr)


if __name__ == '__main__':
    main()