{
  "bootstrap": {
    "unit": "problem_id",
    "repetitions": 2000,
    "seed": 117,
    "confidence": 0.95,
    "method": "paired percentile; omit one-class replicates"
  },
  "panels": {
    "development": {
      "W50": {
        "n_claims": 1649,
        "n_supported": 1431,
        "n_contradicted": 218,
        "n_problems": 60,
        "bootstrap_valid": 2000,
        "auc": {
          "v_self_contrast": {
            "estimate": 0.5272985465992217,
            "ci": [
              0.4761768839017793,
              0.5800232264306209
            ]
          },
          "v_full_contrast": {
            "estimate": 0.6004718583911937,
            "ci": [
              0.5564803686694637,
              0.6433021026919943
            ]
          },
          "v_ao_original_minus_swap": {
            "estimate": 0.5973592599003712,
            "ci": [
              0.5493733631623199,
              0.645674466687874
            ]
          },
          "v_ao_improved_minus_swap": {
            "estimate": 0.6023567275081902,
            "ci": [
              0.5510225626859255,
              0.652724080865441
            ]
          }
        },
        "paired_auc_differences": {
          "v_ao_original_minus_swap_minus_v_self_contrast": {
            "estimate": 0.07006071330114949,
            "ci": [
              0.0316359724413564,
              0.10760241594488736
            ]
          },
          "v_ao_original_minus_swap_minus_v_full_contrast": {
            "estimate": -0.00311259849082246,
            "ci": [
              -0.04946116035946774,
              0.04196378802829727
            ]
          },
          "v_ao_improved_minus_swap_minus_v_self_contrast": {
            "estimate": 0.07505818090896854,
            "ci": [
              0.03678660393801043,
              0.1134308716369462
            ]
          },
          "v_ao_improved_minus_swap_minus_v_full_contrast": {
            "estimate": 0.0018848691169965903,
            "ci": [
              -0.04683181450911347,
              0.04711493764093803
            ]
          }
        }
      },
      "PW50": {
        "n_claims": 1845,
        "n_supported": 1607,
        "n_contradicted": 238,
        "n_problems": 60,
        "bootstrap_valid": 2000,
        "auc": {
          "v_self_contrast": {
            "estimate": 0.5202292491358709,
            "ci": [
              0.47302455048358744,
              0.5616267746296656
            ]
          },
          "v_full_contrast": {
            "estimate": 0.5836701824475901,
            "ci": [
              0.5302556657860417,
              0.6289471958783837
            ]
          },
          "v_ao_original_minus_swap": {
            "estimate": 0.5609622816145748,
            "ci": [
              0.5152479848844068,
              0.6071123914455381
            ]
          },
          "v_ao_improved_minus_swap": {
            "estimate": 0.5665000287607265,
            "ci": [
              0.5158586585759234,
              0.6158716874438612
            ]
          }
        },
        "paired_auc_differences": {
          "v_ao_original_minus_swap_minus_v_self_contrast": {
            "estimate": 0.04073303247870397,
            "ci": [
              0.0021103855503283534,
              0.08331958112034117
            ]
          },
          "v_ao_original_minus_swap_minus_v_full_contrast": {
            "estimate": -0.022707900833015215,
            "ci": [
              -0.07337019508092527,
              0.030310027263651947
            ]
          },
          "v_ao_improved_minus_swap_minus_v_self_contrast": {
            "estimate": 0.046270779624855596,
            "ci": [
              0.0017473298256293517,
              0.09617158799100607
            ]
          },
          "v_ao_improved_minus_swap_minus_v_full_contrast": {
            "estimate": -0.017170153686863587,
            "ci": [
              -0.07163534967341816,
              0.038362516119682964
            ]
          }
        }
      }
    },
    "fresh": {
      "W50": {
        "n_claims": 840,
        "n_supported": 714,
        "n_contradicted": 126,
        "n_problems": 30,
        "bootstrap_valid": 2000,
        "auc": {
          "v_self_contrast": {
            "estimate": 0.5422835800987061,
            "ci": [
              0.4734420100847719,
              0.6054894105892914
            ]
          },
          "v_full_contrast": {
            "estimate": 0.6375772531234716,
            "ci": [
              0.5756638777862576,
              0.6955467394764523
            ]
          },
          "v_ao_original_minus_swap": {
            "estimate": 0.6612311591303188,
            "ci": [
              0.5830722266975259,
              0.7341027274640163
            ]
          },
          "v_ao_improved_minus_swap": {
            "estimate": 0.6383331110221867,
            "ci": [
              0.5680273041804732,
              0.7010477713117174
            ]
          }
        },
        "paired_auc_differences": {
          "v_ao_original_minus_swap_minus_v_self_contrast": {
            "estimate": 0.11894757903161268,
            "ci": [
              0.07570916220201894,
              0.16491499824269792
            ]
          },
          "v_ao_original_minus_swap_minus_v_full_contrast": {
            "estimate": 0.023653906006847247,
            "ci": [
              -0.053512437253838696,
              0.10567466012920965
            ]
          },
          "v_ao_improved_minus_swap_minus_v_self_contrast": {
            "estimate": 0.09604953092348056,
            "ci": [
              0.05956157245427275,
              0.13536233745867501
            ]
          },
          "v_ao_improved_minus_swap_minus_v_full_contrast": {
            "estimate": 0.0007558578987151288,
            "ci": [
              -0.07104343551587859,
              0.06906952631126406
            ]
          }
        }
      }
    }
  }
}
