2025b_fasttrack_source/imc/reports/2024_ktl/test_2_analysis.py

import pickle
import pandas as pd


def get_average_active_secs(df, repeat=1):
    mask = (df > 0.005) & (df < 0.020)
    active = df[mask]
    secs = len(active) / 1000
    return secs / repeat


def analyze_benchmark(benchmark):
    with open(f"output/test_2_{benchmark}_ckpt.pickle", "rb") as f:
        d = pickle.load(f)
    
    d = d.sort_values(by=["time_taken"], ascending=False)
    columns = ["bench_name", "time_taken", "outputs", "is_correct"]


    sla_values = {
        "vFFT": 1.42,
        "vCrc": 1.48,
        "vAes": 1.35,
        "vSha": 1.66,
    }
    sla = sla_values[benchmark]

    print(f"\n+++ benchmark {benchmark} +++")
    print(f"SLA: {sla}")
    print(d[columns])

    mask = d["time_taken"] < sla
    n_total = len(d)
    n_passed = len(d[mask])
    print("")
    print(f"{n_passed} passed (total {n_total})")
    return n_total, n_passed


def do_analysis(benchmarks):
    results = []
    for benchmark in benchmarks:
        try:
            n_total, n_passed = analyze_benchmark(benchmark)
        except:
            continue
        results.append({
            "benchmark": benchmark,
            "total": n_total,
            "passed": n_passed,
            "pass rate (%)": (n_passed / n_total) * 100,
        })
    d = pd.DataFrame(results)

    avg_pass_rate = d["pass rate (%)"].mean()

    print("\n\n+++ summary +++\n")
    print(d)
    print("")
    print(f"average pass rate: {avg_pass_rate:.2f}%")


def main():
    benchmarks = ["vFFT", "vConv2d", "vCrc", "vBasicMath", "vSha", "vStringSearch", "vAes", "vMatMul"]
    benchmarks = ["vFFT", "vCrc", "vAes", "vSha"]
    do_analysis(benchmarks)


if __name__ == "__main__":
    main()