{
  "campaign": "2026-09-21",
  "models": [
    "openai/gpt-oss-20b",
    "qwen/qwen3-30b-a3b"
  ],
  "tasks": [
    "blink-unix",
    "coop-scheduler",
    "frame-corrupt",
    "mqtt-device",
    "noisy-frames",
    "protocol",
    "protocol-retry",
    "uart-echo",
    "uart-menu",
    "watchdog"
  ],
  "pass_at_k": 0.8,
  "pass_at_1": 0.75,
  "pass_at_k_reliability": 0.7,
  "groups": [
    {
      "model": "openai/gpt-oss-20b",
      "task": "blink-unix",
      "attempts": 2,
      "solved": 2,
      "total_iterations": 2,
      "total_duration": 28.450000000000003,
      "infra_failures": 0,
      "success_rate": 1.0,
      "avg_iterations": 1.0,
      "avg_duration": 14.2,
      "passed": true,
      "all_solved": true,
      "incomplete": false,
      "tags": [
        "io"
      ],
      "level": 1
    },
    {
      "model": "openai/gpt-oss-20b",
      "task": "coop-scheduler",
      "attempts": 2,
      "solved": 2,
      "total_iterations": 2,
      "total_duration": 64.88,
      "infra_failures": 0,
      "success_rate": 1.0,
      "avg_iterations": 1.0,
      "avg_duration": 32.4,
      "passed": true,
      "all_solved": true,
      "incomplete": false,
      "tags": [
        "control"
      ],
      "level": 3
    },
    {
      "model": "openai/gpt-oss-20b",
      "task": "frame-corrupt",
      "attempts": 2,
      "solved": 0,
      "total_iterations": 8,
      "total_duration": 692.9,
      "infra_failures": 1,
      "success_rate": 0.0,
      "avg_iterations": 4.0,
      "avg_duration": 346.4,
      "passed": false,
      "all_solved": false,
      "incomplete": false,
      "tags": [
        "resilience"
      ],
      "level": 4
    },
    {
      "model": "openai/gpt-oss-20b",
      "task": "mqtt-device",
      "attempts": 2,
      "solved": 1,
      "total_iterations": 6,
      "total_duration": 422.8,
      "infra_failures": 0,
      "success_rate": 0.5,
      "avg_iterations": 3.0,
      "avg_duration": 211.4,
      "passed": true,
      "all_solved": false,
      "incomplete": false,
      "tags": [
        "protocol"
      ],
      "level": 5
    },
    {
      "model": "openai/gpt-oss-20b",
      "task": "noisy-frames",
      "attempts": 2,
      "solved": 2,
      "total_iterations": 3,
      "total_duration": 54.28,
      "infra_failures": 0,
      "success_rate": 1.0,
      "avg_iterations": 1.5,
      "avg_duration": 27.1,
      "passed": true,
      "all_solved": true,
      "incomplete": false,
      "tags": [
        "resilience",
        "protocol"
      ],
      "level": 3
    },
    {
      "model": "openai/gpt-oss-20b",
      "task": "protocol",
      "attempts": 2,
      "solved": 2,
      "total_iterations": 7,
      "total_duration": 119.16999999999999,
      "infra_failures": 0,
      "success_rate": 1.0,
      "avg_iterations": 3.5,
      "avg_duration": 59.6,
      "passed": true,
      "all_solved": true,
      "incomplete": false,
      "tags": [
        "protocol"
      ],
      "level": 2
    },
    {
      "model": "openai/gpt-oss-20b",
      "task": "protocol-retry",
      "attempts": 2,
      "solved": 2,
      "total_iterations": 3,
      "total_duration": 65.71000000000001,
      "infra_failures": 0,
      "success_rate": 1.0,
      "avg_iterations": 1.5,
      "avg_duration": 32.9,
      "passed": true,
      "all_solved": true,
      "incomplete": false,
      "tags": [
        "protocol",
        "resilience"
      ],
      "level": 3
    },
    {
      "model": "openai/gpt-oss-20b",
      "task": "uart-echo",
      "attempts": 2,
      "solved": 2,
      "total_iterations": 2,
      "total_duration": 15.14,
      "infra_failures": 0,
      "success_rate": 1.0,
      "avg_iterations": 1.0,
      "avg_duration": 7.6,
      "passed": true,
      "all_solved": true,
      "incomplete": false,
      "tags": [
        "io"
      ],
      "level": 1
    },
    {
      "model": "openai/gpt-oss-20b",
      "task": "uart-menu",
      "attempts": 2,
      "solved": 2,
      "total_iterations": 2,
      "total_duration": 31.18,
      "infra_failures": 0,
      "success_rate": 1.0,
      "avg_iterations": 1.0,
      "avg_duration": 15.6,
      "passed": true,
      "all_solved": true,
      "incomplete": false,
      "tags": [
        "fsm"
      ],
      "level": 3
    },
    {
      "model": "openai/gpt-oss-20b",
      "task": "watchdog",
      "attempts": 2,
      "solved": 2,
      "total_iterations": 2,
      "total_duration": 37.57,
      "infra_failures": 0,
      "success_rate": 1.0,
      "avg_iterations": 1.0,
      "avg_duration": 18.8,
      "passed": true,
      "all_solved": true,
      "incomplete": false,
      "tags": [
        "resilience"
      ],
      "level": 3
    },
    {
      "model": "qwen/qwen3-30b-a3b",
      "task": "blink-unix",
      "attempts": 2,
      "solved": 2,
      "total_iterations": 2,
      "total_duration": 151.55,
      "infra_failures": 0,
      "success_rate": 1.0,
      "avg_iterations": 1.0,
      "avg_duration": 75.8,
      "passed": true,
      "all_solved": true,
      "incomplete": false,
      "tags": [
        "io"
      ],
      "level": 1
    },
    {
      "model": "qwen/qwen3-30b-a3b",
      "task": "coop-scheduler",
      "attempts": 2,
      "solved": 2,
      "total_iterations": 5,
      "total_duration": 1889.56,
      "infra_failures": 0,
      "success_rate": 1.0,
      "avg_iterations": 2.5,
      "avg_duration": 944.8,
      "passed": true,
      "all_solved": true,
      "incomplete": false,
      "tags": [
        "control"
      ],
      "level": 3
    },
    {
      "model": "qwen/qwen3-30b-a3b",
      "task": "frame-corrupt",
      "attempts": 2,
      "solved": 0,
      "total_iterations": 10,
      "total_duration": 3240.24,
      "infra_failures": 0,
      "success_rate": 0.0,
      "avg_iterations": 5.0,
      "avg_duration": 1620.1,
      "passed": false,
      "all_solved": false,
      "incomplete": false,
      "tags": [
        "resilience"
      ],
      "level": 4
    },
    {
      "model": "qwen/qwen3-30b-a3b",
      "task": "mqtt-device",
      "attempts": 2,
      "solved": 0,
      "total_iterations": 10,
      "total_duration": 3516.3599999999997,
      "infra_failures": 0,
      "success_rate": 0.0,
      "avg_iterations": 5.0,
      "avg_duration": 1758.2,
      "passed": false,
      "all_solved": false,
      "incomplete": false,
      "tags": [
        "protocol"
      ],
      "level": 5
    },
    {
      "model": "qwen/qwen3-30b-a3b",
      "task": "noisy-frames",
      "attempts": 2,
      "solved": 2,
      "total_iterations": 8,
      "total_duration": 1403.49,
      "infra_failures": 0,
      "success_rate": 1.0,
      "avg_iterations": 4.0,
      "avg_duration": 701.7,
      "passed": true,
      "all_solved": true,
      "incomplete": false,
      "tags": [
        "resilience",
        "protocol"
      ],
      "level": 3
    },
    {
      "model": "qwen/qwen3-30b-a3b",
      "task": "protocol",
      "attempts": 2,
      "solved": 0,
      "total_iterations": 10,
      "total_duration": 1163.6100000000001,
      "infra_failures": 0,
      "success_rate": 0.0,
      "avg_iterations": 5.0,
      "avg_duration": 581.8,
      "passed": false,
      "all_solved": false,
      "incomplete": false,
      "tags": [
        "protocol"
      ],
      "level": 2
    },
    {
      "model": "qwen/qwen3-30b-a3b",
      "task": "protocol-retry",
      "attempts": 2,
      "solved": 1,
      "total_iterations": 9,
      "total_duration": 1869.0900000000001,
      "infra_failures": 0,
      "success_rate": 0.5,
      "avg_iterations": 4.5,
      "avg_duration": 934.5,
      "passed": true,
      "all_solved": false,
      "incomplete": false,
      "tags": [
        "protocol",
        "resilience"
      ],
      "level": 3
    },
    {
      "model": "qwen/qwen3-30b-a3b",
      "task": "uart-echo",
      "attempts": 2,
      "solved": 2,
      "total_iterations": 2,
      "total_duration": 128.15,
      "infra_failures": 0,
      "success_rate": 1.0,
      "avg_iterations": 1.0,
      "avg_duration": 64.1,
      "passed": true,
      "all_solved": true,
      "incomplete": false,
      "tags": [
        "io"
      ],
      "level": 1
    },
    {
      "model": "qwen/qwen3-30b-a3b",
      "task": "uart-menu",
      "attempts": 2,
      "solved": 2,
      "total_iterations": 6,
      "total_duration": 786.44,
      "infra_failures": 0,
      "success_rate": 1.0,
      "avg_iterations": 3.0,
      "avg_duration": 393.2,
      "passed": true,
      "all_solved": true,
      "incomplete": false,
      "tags": [
        "fsm"
      ],
      "level": 3
    },
    {
      "model": "qwen/qwen3-30b-a3b",
      "task": "watchdog",
      "attempts": 2,
      "solved": 2,
      "total_iterations": 2,
      "total_duration": 395.65,
      "infra_failures": 0,
      "success_rate": 1.0,
      "avg_iterations": 1.0,
      "avg_duration": 197.8,
      "passed": true,
      "all_solved": true,
      "incomplete": false,
      "tags": [
        "resilience"
      ],
      "level": 3
    }
  ],
  "class_profile": {
    "openai/gpt-oss-20b": {
      "control": {
        "attempts": 2,
        "solved": 2,
        "success_rate": 1.0
      },
      "fsm": {
        "attempts": 2,
        "solved": 2,
        "success_rate": 1.0
      },
      "io": {
        "attempts": 4,
        "solved": 4,
        "success_rate": 1.0
      },
      "protocol": {
        "attempts": 8,
        "solved": 7,
        "success_rate": 0.88
      },
      "resilience": {
        "attempts": 7,
        "solved": 6,
        "success_rate": 0.86
      }
    },
    "qwen/qwen3-30b-a3b": {
      "control": {
        "attempts": 2,
        "solved": 2,
        "success_rate": 1.0
      },
      "fsm": {
        "attempts": 2,
        "solved": 2,
        "success_rate": 1.0
      },
      "io": {
        "attempts": 4,
        "solved": 4,
        "success_rate": 1.0
      },
      "protocol": {
        "attempts": 8,
        "solved": 3,
        "success_rate": 0.38
      },
      "resilience": {
        "attempts": 8,
        "solved": 5,
        "success_rate": 0.62
      }
    }
  },
  "unparseable": []
}