bookworm-smart-assistant/scripts/skill-effectiveness.js

/* eslint-disable no-console */
'use strict';

const fs   = require('fs');
const path = require('path');

// ── 路径常量 ─────────────────────────────────────────────────────────────────
const CLAUDE_ROOT = path.resolve(__dirname, '..');
const DEBUG_DIR   = path.join(CLAUDE_ROOT, 'debug');

// ── JSONL 工具函数 ────────────────────────────────────────────────────────────
/** 安全解析单文件中的每一行 JSONL，遇到损坏行跳过 */
function readJsonl(filePath) {
  try {
    return fs.readFileSync(filePath, 'utf8')
      .split('\n')
      .filter(Boolean)
      .map(line => { try { return JSON.parse(line); } catch { return null; } })
      .filter(Boolean);
  } catch {
    return [];
  }
}

/** 读取 DEBUG_DIR 下所有匹配 glob 前缀的 .jsonl 文件 */
function readGlobJsonl(prefix) {
  try {
    return fs.readdirSync(DEBUG_DIR)
      .filter(f => f.startsWith(prefix) && f.endsWith('.jsonl'))
      .flatMap(f => readJsonl(path.join(DEBUG_DIR, f)));
  } catch {
    return [];
  }
}

// ── 日期截止计算 ──────────────────────────────────────────────────────────────
function cutoffTs(days) {
  const d = new Date();
  d.setDate(d.getDate() - days);
  return d.toISOString();
}

// ── 核心分析函数 ──────────────────────────────────────────────────────────────
/**
 * @param {{ days?: number, save?: boolean, json?: boolean }} opts
 * @returns {{ skills: object[], summary: object }}
 */
function analyze({ days = 7, save = false } = {}) {
  const since = cutoffTs(days);

  // 1. 读取路由数据（route-stats-daily-*.jsonl）
  const routeRows = readGlobJsonl('route-stats-daily-')
    .filter(r => r.ts >= since && r.skill);

  // 2. 读取 outcome 数据（skill-outcome.jsonl）
  const outcomeRows = readJsonl(path.join(DEBUG_DIR, 'skill-outcome.jsonl'))
    .filter(r => r.ts >= since && r.skill);

  // 3. 读取 activity skill 事件（activity-*.jsonl，event === 'skill'）
  const activityRows = readGlobJsonl('activity-')
    .filter(r => r.ts >= since && r.event === 'skill' && r.detail);

  // 4. 汇总每个 Skill 的指标
  const skillMap = {};

  function getOrCreate(name) {
    if (!skillMap[name]) {
      skillMap[name] = { routeCount: 0, mustInvokeCount: 0, actualInvocations: 0,
                         successCount: 0, outcomeCount: 0 };
    }
    return skillMap[name];
  }

  for (const r of routeRows) {
    const s = getOrCreate(r.skill);
    s.routeCount++;
    if (r.mustInvoke) s.mustInvokeCount++;
  }

  for (const r of outcomeRows) {
    const s = getOrCreate(r.skill);
    s.outcomeCount++;
    if (r.success) s.successCount++;
  }

  for (const r of activityRows) {
    getOrCreate(r.detail).actualInvocations++;
  }

  // 5. 计算派生指标
  const allRouteCounts = Object.values(skillMap).map(s => s.routeCount);
  const maxRouteCount  = Math.max(...allRouteCounts, 1); // 避免除零

  const skills = Object.entries(skillMap).map(([name, s]) => {
    // 合规率：实际调用 / MUST_INVOKE 路由次数（无强制路由时用实际/总路由）
    const complianceDenom  = s.mustInvokeCount > 0 ? s.mustInvokeCount : s.routeCount;
    const invocationCompliance = complianceDenom > 0
      ? Math.min(s.actualInvocations / complianceDenom, 1)
      : 0;

    // 成功率：有 outcome 数据才计算，否则用 0.5 作中性值
    const successRate = s.outcomeCount > 0
      ? s.successCount / s.outcomeCount
      : 0.5;

    // 路由量归一化
    const normalizedRoute = s.routeCount / maxRouteCount;

    // 综合分（满分 100）
    const overall = (invocationCompliance * 0.5 + successRate * 0.3 + normalizedRoute * 0.2) * 100;

    return {
      name,
      routeCount:           s.routeCount,
      mustInvokeCount:      s.mustInvokeCount,
      actualInvocations:    s.actualInvocations,
      outcomeCount:         s.outcomeCount,
      invocationCompliance: parseFloat((invocationCompliance * 100).toFixed(1)),
      successRate:          parseFloat((successRate * 100).toFixed(1)),
      overall:              parseFloat(overall.toFixed(1)),
    };
  });

  // 按综合分降序排列
  skills.sort((a, b) => b.overall - a.overall);

  const summary = {
    days,
    totalRoutes:       routeRows.length,
    mustInvokeRoutes:  routeRows.filter(r => r.mustInvoke).length,
    actualInvocations: activityRows.length,
  };

  const result = { skills, summary };

  // 6. 可选持久化
  if (save) {
    const outPath = path.join(DEBUG_DIR, 'skill-effectiveness-report.json');
    fs.writeFileSync(outPath, JSON.stringify(result, null, 2), 'utf8');
    console.error(`[saved] ${outPath}`);
  }

  return result;
}

// ── 人类可读输出 ──────────────────────────────────────────────────────────────
function printReport({ skills, summary }) {
  const { days, totalRoutes, mustInvokeRoutes, actualInvocations } = summary;

  console.log('\n═══ Skill Effectiveness Report ═══');
  console.log(`分析窗口: 最近 ${days} 天`);
  console.log(`总路由: ${totalRoutes} | MUST_INVOKE: ${mustInvokeRoutes} | 实际调用: ${actualInvocations}\n`);

  if (skills.length === 0) {
    console.log('（无数据）');
    return;
  }

  // 表头
  const hdr = '排名  技能名                        路由  强制  实际调用  合规率    成功率    综合分';
  console.log(hdr);
  console.log('─'.repeat(hdr.length));

  skills.forEach((s, i) => {
    const rank     = String(i + 1).padStart(3) + '.';
    const name     = s.name.padEnd(30);
    const route    = String(s.routeCount).padStart(4);
    const must     = String(s.mustInvokeCount).padStart(4);
    const actual   = String(s.actualInvocations).padStart(8);
    const comply   = `${s.invocationCompliance}%`.padStart(8);
    const success  = `${s.successRate}%`.padStart(8);
    const score    = String(s.overall).padStart(8);
    console.log(`${rank}  ${name}${route}  ${must}${actual}  ${comply}  ${success}  ${score}`);
  });

  console.log('');
}

// ── CLI 入口 ──────────────────────────────────────────────────────────────────
if (require.main === module) {
  const args = process.argv.slice(2);
  const days = parseInt(args.find((a, i, arr) => arr[i - 1] === '--days') || '7');
  const json = args.includes('--json');
  const save = args.includes('--save');
  const result = analyze({ days, save });
  if (json) console.log(JSON.stringify(result, null, 2));
  else printReport(result);
}

module.exports = { analyze };