[ { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps@recall0.90", "system": "hnswlib", "value": 9480.442447565747, "ci": [ 9417.422568871087, 9697.221801401593 ], "n": 3, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.serve_rounds[*].configs[2]", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "9,480" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps@recall0.90", "system": "hms", "value": 23293.629504996767, "ci": [ 22814.748394915856, 23733.858833120175 ], "n": 3, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[*].configs[11]", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "23,294" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps@recall0.90", "derived": true, "warm": "hms", "other": "hnswlib", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps@recall0.95", "system": "hnswlib", "value": 7630.414893330864, "ci": [ 7474.952830643235, 7814.989707482342 ], "n": 3, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.serve_rounds[*].configs[3]", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "7,630" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps@recall0.95", "system": "hms", "value": 17480.964142532004, "ci": [ 15253.129845587078, 17833.849020964924 ], "n": 3, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[*].configs[2]", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "17,481" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps@recall0.95", "derived": true, "warm": "hms", "other": "hnswlib", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps@recall0.99", "system": "hnswlib", "value": 3192.520978113118, "ci": [ 3080.7832588347464, 3298.7230563365106 ], "n": 3, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.serve_rounds[*].configs[6]", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "3,193" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps@recall0.99", "system": "hms", "value": 8794.271294386524, "ci": [ 8591.235496775094, 9082.365742451297 ], "n": 3, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[*].configs[5]", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "8,794" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps@recall0.99", "derived": true, "warm": "hms", "other": "hnswlib", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency@recall0.95", "system": "hnswlib", "value": 133.32475, "ci": [ 128.75775, 136.15449999999998 ], "n": 3, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.serve_rounds[*].configs[3]", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "133" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency@recall0.95", "system": "hms", "value": 62.091, "ci": [ 55.067, 62.514 ], "n": 3, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[*].configs[2]", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "62" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency@recall0.95", "derived": true, "warm": "hms", "other": "hnswlib", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency@recall0.95", "system": "hnswlib", "value": 219.73541500000005, "ci": [ 216.20628, 228.29286000000002 ], "n": 3, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.serve_rounds[*].configs[3]", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "220" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency@recall0.95", "system": "hms", "value": 104.57, "ci": [ 84.128, 110.87700000000001 ], "n": 3, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[*].configs[2]", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "105" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency@recall0.95", "derived": true, "warm": "hms", "other": "hnswlib", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "hnswlib", "value": 182.4948315215006, "ci": [ 182.2122630720005, 182.7773999710007 ], "n": 2, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "182.49" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "hms", "value": 312.6031632165, "ci": [ 310.021226209, 315.185100224 ], "n": 2, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[graph_built]", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "312.60" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "derived": true, "warm": "hms", "other": "hnswlib", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "hnswlib", "value": 1524.958984375, "ci": [ 1520.74609375, 1529.171875 ], "n": 2, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "1,525" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "hms", "value": 2967.65234375, "ci": [ 2957.69140625, 2977.61328125 ], "n": 2, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[graph_built]", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "2,968" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "derived": true, "warm": "hms", "other": "hnswlib", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "hnswlib", "value": 1019.3671875, "ci": [ 1018.99609375, 1019.375 ], "n": 3, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.serve_rounds[*].rss_steady", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "1,019" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "hms", "value": 2203.828125, "ci": [ 1261.109375, 2223.64453125 ], "n": 3, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[*].rss_steady", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "2,204" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "derived": true, "warm": "hms", "other": "hnswlib", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "index_bytes_mib", "system": "hnswlib", "value": 873.813777923584, "ci": [ 873.813777923584, 873.813777923584 ], "n": 1, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "873.8" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "index_bytes_mib", "system": "hms", "value": 446.3023262023926, "ci": [ 446.3023262023926, 446.3023262023926 ], "n": 1, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "446.3" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "index_bytes_mib", "derived": true, "warm": "hms", "other": "hnswlib", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "hnswlib", "value": 0.748887823, "ci": [ 0.536470403, 0.77115552 ], "n": 3, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.builds.m48e500.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.75" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "hms", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.builds.vertex-d32.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "n/m" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "derived": true, "warm": "hms", "other": "hnswlib", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/m" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "hnswlib", "value": 6.010148169989407, "ci": [ 6.010148169989407, 6.010148169989407 ], "n": 1, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "6.010" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "hms", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "n/m" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "derived": true, "warm": "hms", "other": "hnswlib", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/m" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "hnswlib", "value": 3.3136410399856686, "ci": [ 3.3136410399856686, 3.3136410399856686 ], "n": 1, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "3.314" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "hms", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "n/m" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "derived": true, "warm": "hms", "other": "hnswlib", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/m" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "hnswlib", "value": 61.750614166259766, "ci": [ 61.750614166259766, 61.750614166259766 ], "n": 1, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "61.8" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "hms", "value": 3.378631591796875, "ci": [ 3.378631591796875, 3.378631591796875 ], "n": 1, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "3.4" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "derived": true, "warm": "hms", "other": "hnswlib", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "hnswlib", "value": 2, "ci": [ 2, 2 ], "n": 1, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hnswlib.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "2" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "hms", "value": 92, "ci": [ 92, 92 ], "n": 1, "source": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install._crate_dependencies", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "92" }, { "section": "sift-128-euclidean (1,000,000 vectors, 10,000 queries)", "file": "benchmarks/results/competitive/heavy-ann-sift-128-euclidean-n1000000-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "derived": true, "warm": "hms", "other": "hnswlib", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "hms", "value": 0.26537791112001474, "ci": [ 0.23702131021485068, 0.2947007216218275 ], "n": 500, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.2654" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "sqlite-fts5", "value": 0.26781574230549227, "ci": [ 0.2371816120958971, 0.2968809113532711 ], "n": 500, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.2678" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "lucene", "value": 0.2656768159122737, "ci": [ 0.23666905387824008, 0.2935669445083154 ], "n": 500, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.2657" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "tantivy", "value": 0.26427698472308014, "ci": [ 0.23619095893739991, 0.29259476476656954 ], "n": 500, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.2643" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "hms", "value": 0.5422345043845044, "ci": [ 0.5079547619047619, 0.5755660173160173 ], "n": 500, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.5422" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "sqlite-fts5", "value": 0.5545597791097792, "ci": [ 0.5197631368631369, 0.5880408036408037 ], "n": 500, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.5546" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "lucene", "value": 0.5444845043845044, "ci": [ 0.5103370795870796, 0.5756238816738817 ], "n": 500, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.5445" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "tantivy", "value": 0.5405661283161284, "ci": [ 0.5067542235542235, 0.5743550172050172 ], "n": 500, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.5406" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "hms", "value": 3611.9250624371816, "ci": [ 3305.9404867767275, 3636.4703502390053 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "3,612" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "sqlite-fts5", "value": 26.391160458321984, "ci": [ 25.512147136220875, 26.854216080674412 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "26" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "lucene", "value": 1267.696321226662, "ci": [ 1136.4776689832, 1286.2379536098217 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "1,268" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "tantivy", "value": 1617.0952478997635, "ci": [ 1470.224415289986, 1651.386284785746 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "1,617" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "hms", "value": 269.099, "ci": [ 266.955, 293.864 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "269" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "sqlite-fts5", "value": 34691.752, "ci": [ 34254.818, 35827.793 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "34,692" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "lucene", "value": 721.343, "ci": [ 713.381, 802.49 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "721" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "tantivy", "value": 600.18, "ci": [ 598.118, 668.511 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "600" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "hms", "value": 508.777, "ci": [ 460.002, 565.727 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "509" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "sqlite-fts5", "value": 88406.225, "ci": [ 87276.419, 94275.323 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "88,406" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "lucene", "value": 1754.898, "ci": [ 1671.394, 2145.252 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "1,755" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "tantivy", "value": 1269.343, "ci": [ 1093.41, 1423.457 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "1,269" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "hms", "value": 2.156009168, "ci": [ 2.131987949, 2.188588237 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "2.16" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "sqlite-fts5", "value": 1.1806630159990164, "ci": [ 1.1729135619989393, 1.1811330580003414 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "1.18" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "lucene", "value": 2.630518981, "ci": [ 2.473350822, 2.778356151 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "2.63" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "tantivy", "value": 2.6500262409990682, "ci": [ 2.6409532399993623, 2.6505701259993657 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "2.65" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "hms", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "sqlite-fts5", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "lucene", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "tantivy", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "hms", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].peak_rss_bytes", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "sqlite-fts5", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].peak_rss_bytes", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "lucene", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].peak_rss_bytes", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "tantivy", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].peak_rss_bytes", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "hms", "value": 20.562817573547363, "ci": [ 20.562817573547363, 20.562817573547363 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "20.6" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "sqlite-fts5", "value": 27.0859375, "ci": [ 27.0859375, 27.0859375 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "27.1" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "lucene", "value": 12.16652774810791, "ci": [ 12.16652774810791, 12.16652774810791 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "12.2" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "tantivy", "value": 17.35812759399414, "ci": [ 17.35812759399414, 17.35812759399414 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "17.4" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "derived": true, "warm": "hms", "other": "lucene", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "hms", "value": 0.840599199, "ci": [ 0.836939676, 0.942827962 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.84" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "sqlite-fts5", "value": 0.081541718, "ci": [ 0.077916238, 0.088280834 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.08" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "lucene", "value": 0.291951304, "ci": [ 0.275507479, 0.31941395 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.29" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "tantivy", "value": 0.024903889, "ci": [ 0.023638664, 0.02968958 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.02" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "hms", "value": 1.71895986, "ci": [ 1.71895986, 1.71895986 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "1.719" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "sqlite-fts5", "value": 9.141574070017668, "ci": [ 9.141574070017668, 9.141574070017668 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "9.142" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "lucene", "value": 31.037972089999997, "ci": [ 31.037972089999997, 31.037972089999997 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "31.038" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "tantivy", "value": 82.71144980999452, "ci": [ 82.71144980999452, 82.71144980999452 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "82.711" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "hms", "value": 3.75641631, "ci": [ 3.75641631, 3.75641631 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "3.756" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "sqlite-fts5", "value": 8.763046479998593, "ci": [ 8.763046479998593, 8.763046479998593 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "8.763" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "lucene", "value": 14.948087370000001, "ci": [ 14.948087370000001, 14.948087370000001 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "14.948" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "tantivy", "value": 43.996110599982785, "ci": [ 43.996110599982785, 43.996110599982785 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "43.996" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "hms", "value": 2.4908218383789062, "ci": [ 2.4908218383789062, 2.4908218383789062 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "2.5" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "sqlite-fts5", "value": 0.0, "ci": [ 0.0, 0.0 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "0.0" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "lucene", "value": 5.715169906616211, "ci": [ 5.715169906616211, 5.715169906616211 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "5.7" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "tantivy", "value": 13.801004409790039, "ci": [ 13.801004409790039, 13.801004409790039 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "13.8" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "hms", "value": 92, "ci": [ 92, 92 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install._crate_dependencies", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "92" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "sqlite-fts5", "value": 0, "ci": [ 0, 0 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "0" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "lucene", "value": 2, "ci": [ 2, 2 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "2" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "tantivy", "value": 1, "ci": [ 1, 1 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "1" }, { "section": "beir:fiqa (57,600 documents, 500 queries)", "file": "benchmarks/results/competitive/standard-text-beir-fiqa-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "hms", "value": 0.4561419499447157, "ci": [ 0.34378214839170335, 0.5689042043949878 ], "n": 100, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.4561" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "hms-eager", "value": 0.4561419499447157, "ci": [ 0.34378214839170335, 0.5689042043949878 ], "n": 100, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.4561" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "sqlite-fts5", "value": 0.45009623895127726, "ci": [ 0.3322770359003301, 0.5667724678193115 ], "n": 100, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.4501" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "lucene", "value": 0.46265299364569407, "ci": [ 0.3452561205276061, 0.5757421844639584 ], "n": 100, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.4627" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "tantivy", "value": 0.4491250108653578, "ci": [ 0.3343985674609109, 0.5627609557586442 ], "n": 100, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.4491" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "whoosh", "value": 0.4401216437164885, "ci": [ 0.3277974760341089, 0.5514643172818812 ], "n": 100, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.4401" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "bm25s", "value": 0.44175557833638, "ci": [ 0.3275175531106332, 0.5507361325996123 ], "n": 100, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.4418" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "derived": true, "warm": "hms", "other": "lucene", "factor": 0.9859267230723582, "factor_infinite": false, "ci": [ 0.597111272490446, 1.6477744218570605 ], "verdict": "TIE", "text": "0.99x" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "hms", "value": 0.8203333333333334, "ci": [ 0.7254999999999999, 0.905 ], "n": 100, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.8203" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "hms-eager", "value": 0.8203333333333334, "ci": [ 0.7254999999999999, 0.905 ], "n": 100, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.8203" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "sqlite-fts5", "value": 0.8391666666666667, "ci": [ 0.7396666666666667, 0.919 ], "n": 100, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.8392" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "lucene", "value": 0.8236666666666665, "ci": [ 0.7254999999999999, 0.905 ], "n": 100, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.8237" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "tantivy", "value": 0.8203333333333334, "ci": [ 0.7254999999999999, 0.905 ], "n": 100, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.8203" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "whoosh", "value": 0.8206666666666667, "ci": [ 0.7238333333333334, 0.909 ], "n": 100, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.8207" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "bm25s", "value": 0.8311666666666666, "ci": [ 0.733, 0.915 ], "n": 100, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.8312" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 0.9775571002979145, "factor_infinite": false, "ci": [ 0.7894450489662675, 1.2235241099594412 ], "verdict": "TIE", "text": "0.98x" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "hms", "value": 2288.679292332942, "ci": [ 2118.0838077046783, 2365.9257667604606 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "2,289" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "hms-eager", "value": 2232.159244574398, "ci": [ 2072.383935982237, 2305.5221197210144 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "2,232" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "sqlite-fts5", "value": 32.425558442954014, "ci": [ 31.981502390253354, 32.80521041929057 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "32" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "lucene", "value": 1002.2853725087555, "ci": [ 958.5533420939997, 1019.166381826649 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "1,002" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "tantivy", "value": 2503.8034244261294, "ci": [ 2269.622326220352, 2556.782210084709 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "2,504" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "whoosh", "value": 13.599709372345025, "ci": [ 13.408249665024112, 13.69029410190459 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "14" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "bm25s", "value": 1102.183527988817, "ci": [ 1029.9761689263914, 1149.0237365783112 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "1,102" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "derived": true, "warm": "hms", "other": "tantivy", "factor": 0.914081061638258, "factor_infinite": false, "ci": [ 0.8284177664215303, 1.0424314827306465 ], "verdict": "TIE", "text": "0.91x" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "hms", "value": 420.2165, "ci": [ 401.735, 446.873 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "420" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "hms-eager", "value": 431.558, "ci": [ 411.905, 461.791 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "432" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "sqlite-fts5", "value": 24740.3205, "ci": [ 24537.211, 25147.863 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "24,740" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "lucene", "value": 930.0550000000001, "ci": [ 916.458, 947.398 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "930" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "tantivy", "value": 379.59749999999997, "ci": [ 372.469, 416.904 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "380" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "whoosh", "value": 68913.053, "ci": [ 67785.599, 69688.196 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "68,913" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "bm25s", "value": 899.6585, "ci": [ 848.974, 947.889 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "900" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "derived": true, "warm": "hms", "other": "tantivy", "factor": 0.9033379222377036, "factor_infinite": false, "ci": [ 0.8335007932902637, 1.0377587215452972 ], "verdict": "TIE", "text": "0.90x" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "hms", "value": 774.961, "ci": [ 725.49, 802.218 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "775" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "hms-eager", "value": 780.2764999999999, "ci": [ 731.141, 815.193 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "780" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "sqlite-fts5", "value": 118386.10500000001, "ci": [ 117000.749, 119354.506 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "118,386" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "lucene", "value": 2212.803, "ci": [ 1854.138, 2609.766 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "2,213" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "tantivy", "value": 720.3705, "ci": [ 692.567, 741.16 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "720" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "whoosh", "value": 131749.15350000001, "ci": [ 129717.812, 132579.185 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "131,749" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "bm25s", "value": 1249.0225, "ci": [ 1130.652, 1301.768 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "1,249" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "derived": true, "warm": "hms", "other": "tantivy", "factor": 0.9295571002927889, "factor_infinite": false, "ci": [ 0.8633152085842004, 1.0215991950268095 ], "verdict": "TIE", "text": "0.93x" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "hms", "value": 2.515448795, "ci": [ 2.452041575, 2.578856015 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "2.52" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "hms-eager", "value": 16.573853896499998, "ci": [ 16.402460064, 16.745247729 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "16.57" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "sqlite-fts5", "value": 2.0610938575000546, "ci": [ 2.032452352000064, 2.0897353630000453 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "2.06" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "lucene", "value": 4.0917769785, "ci": [ 4.042061258, 4.141492699 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "4.09" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "tantivy", "value": 3.476251229000013, "ci": [ 3.4579393139999866, 3.4945631440000398 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "3.48" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "whoosh", "value": 41.11348974550003, "ci": [ 40.822708994000095, 41.40427049699997 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "41.11" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "bm25s", "value": 5.152338578999888, "ci": [ 5.13049070399984, 5.174186453999937 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "5.15" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 0.8193742053493299, "factor_infinite": false, "ci": [ 0.7881216865843764, 0.8522430387421328 ], "verdict": "LOSS", "text": "0.82x" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "hms", "value": 132.7890625, "ci": [ 132.38671875, 133.19140625 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "133" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "hms-eager", "value": 166.75390625, "ci": [ 165.9453125, 167.5625 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "167" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "sqlite-fts5", "value": 120.546875, "ci": [ 120.5390625, 120.5546875 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "121" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "lucene", "value": 768.109375, "ci": [ 764.265625, 771.953125 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "768" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "tantivy", "value": 244.4921875, "ci": [ 244.26171875, 244.72265625 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "244" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "whoosh", "value": 536.255859375, "ci": [ 536.08984375, 536.421875 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "536" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "bm25s", "value": 236.708984375, "ci": [ 236.12890625, 237.2890625 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "237" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 0.9078072601047243, "factor_infinite": false, "ci": [ 0.9050063055400769, 0.9106252397391638 ], "verdict": "LOSS", "text": "0.91x" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "hms", "value": 65.912109375, "ci": [ 64.90234375, 66.77734375 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "66" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "hms-eager", "value": 74.197265625, "ci": [ 73.5625, 75.1015625 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "74" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "sqlite-fts5", "value": 25.828125, "ci": [ 25.05078125, 26.326171875 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "26" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "lucene", "value": 438.154296875, "ci": [ 434.078125, 439.94140625 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "438" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "tantivy", "value": 37.724609375, "ci": [ 37.06640625, 38.5703125 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "38" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "whoosh", "value": 141.31640625, "ci": [ 140.01953125, 143.0390625 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "141" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "bm25s", "value": 83.640625, "ci": [ 83.04296875, 84.0703125 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "84" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 0.39185705396035203, "factor_infinite": false, "ci": [ 0.3751389295115531, 0.40562744507974724 ], "verdict": "LOSS", "text": "0.39x" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "hms", "value": 9.793286323547363, "ci": [ 9.793286323547363, 9.793286323547363 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "9.8" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "hms-eager", "value": 9.785473823547363, "ci": [ 9.785473823547363, 9.785473823547363 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "9.8" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "sqlite-fts5", "value": 63.0390625, "ci": [ 63.0390625, 63.0390625 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "63.0" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "lucene", "value": 10.313970565795898, "ci": [ 10.313970565795898, 10.313970565795898 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "10.3" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "tantivy", "value": 13.706984519958496, "ci": [ 13.706984519958496, 13.706984519958496 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "13.7" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "whoosh", "value": 48.566134452819824, "ci": [ 48.566134452819824, 48.566134452819824 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "48.6" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "bm25s", "value": 14.216367721557617, "ci": [ 14.216367721557617, 14.216367721557617 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "14.2" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "derived": true, "warm": "hms", "other": "lucene", "factor": 1.053167468513259, "factor_infinite": false, "ci": null, "verdict": "WIN", "text": "1.05x" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "hms", "value": 0.619563115, "ci": [ 0.594086714, 0.641814565 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.62" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "hms-eager", "value": 0.6272474809999999, "ci": [ 0.591178799, 0.649707937 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.63" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "sqlite-fts5", "value": 0.08885403750000001, "ci": [ 0.086029671, 0.092307714 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.09" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "lucene", "value": 0.4474606435, "ci": [ 0.433207579, 0.479147153 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.45" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "tantivy", "value": 0.0718216415, "ci": [ 0.065463099, 0.077803712 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.07" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "whoosh", "value": 0.289982277, "ci": [ 0.279255388, 0.305334852 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.29" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "bm25s", "value": 0.24022912549999997, "ci": [ 0.224627369, 0.248742769 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.24" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "derived": true, "warm": "hms", "other": "tantivy", "factor": 0.11592304280412175, "factor_infinite": false, "ci": [ 0.10199690466669294, 0.1309635616594516 ], "verdict": "LOSS", "text": "0.12x" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "hms", "value": 3.9187444300000003, "ci": [ 3.9187444300000003, 3.9187444300000003 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "3.919" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "hms-eager", "value": 3.8050166800000005, "ci": [ 3.8050166800000005, 3.8050166800000005 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "3.805" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "sqlite-fts5", "value": 4.625860979999743, "ci": [ 4.625860979999743, 4.625860979999743 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "4.626" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "lucene", "value": 18.03878455, "ci": [ 18.03878455, 18.03878455 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "18.039" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "tantivy", "value": 163.8972191999983, "ci": [ 163.8972191999983, 163.8972191999983 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "163.897" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "whoosh", "value": 146.63095731999874, "ci": [ 146.63095731999874, 146.63095731999874 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "146.631" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "bm25s", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.ops", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "n/m" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 1.1804446711519134, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "1.18x" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "hms", "value": 3.1620943699999997, "ci": [ 3.1620943699999997, 3.1620943699999997 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "3.162" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "hms-eager", "value": 3.2690420799999997, "ci": [ 3.2690420799999997, 3.2690420799999997 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "3.269" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "sqlite-fts5", "value": 6.417760269996506, "ci": [ 6.417760269996506, 6.417760269996506 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "6.418" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "lucene", "value": 1.48195727, "ci": [ 1.48195727, 1.48195727 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "1.482" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "tantivy", "value": 176.53093911999804, "ci": [ 176.53093911999804, 176.53093911999804 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "176.531" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "whoosh", "value": 16.973000140001204, "ci": [ 16.973000140001204, 16.973000140001204 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "16.973" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "bm25s", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.ops", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "n/m" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "derived": true, "warm": "hms", "other": "lucene", "factor": 0.4686632012187543, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "0.47x" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "hms", "value": 2.3764114379882812, "ci": [ 2.3764114379882812, 2.3764114379882812 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "2.4" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "hms-eager", "value": 2.3764114379882812, "ci": [ 2.3764114379882812, 2.3764114379882812 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "2.4" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "sqlite-fts5", "value": 0.0, "ci": [ 0.0, 0.0 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.install", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "0.0" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "lucene", "value": 5.715169906616211, "ci": [ 5.715169906616211, 5.715169906616211 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.install", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "5.7" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "tantivy", "value": 13.801004409790039, "ci": [ 13.801004409790039, 13.801004409790039 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.install", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "13.8" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "whoosh", "value": 1.9728384017944336, "ci": [ 1.9728384017944336, 1.9728384017944336 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.install", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "2.0" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "bm25s", "value": 56.54034996032715, "ci": [ 56.54034996032715, 56.54034996032715 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.install", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "56.5" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "derived": true, "warm": "hms", "other": "whoosh", "factor": 0.8301754360619107, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "0.83x" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "hms", "value": 92, "ci": [ 92, 92 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms.install._crate_dependencies", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "92" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "hms-eager", "value": 92, "ci": [ 92, 92 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.hms-eager.install._crate_dependencies", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "92" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "sqlite-fts5", "value": 0, "ci": [ 0, 0 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "0" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "lucene", "value": 2, "ci": [ 2, 2 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.lucene.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "2" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "tantivy", "value": 1, "ci": [ 1, 1 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.tantivy.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "1" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "whoosh", "value": 2, "ci": [ 2, 2 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.whoosh.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "2" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "bm25s", "value": 3, "ci": [ 3, 3 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json#systems.bm25s.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "3" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/heavy-text-memory-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "derived": true, "warm": "hms", "other": "tantivy", "factor": 0.010869565217391304, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "0.01x" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "hms", "value": 0.4561419499447157, "ci": [ 0.37855253576415726, 0.5350347701184234 ], "n": 100, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.4561" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "sqlite-fts5", "value": 0.45009623895127726, "ci": [ 0.3664069435962895, 0.5334244252772741 ], "n": 100, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.4501" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "lucene", "value": 0.46265299364569407, "ci": [ 0.38441067792846956, 0.5426832663257491 ], "n": 100, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.4627" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "tantivy", "value": 0.4491250108653578, "ci": [ 0.372011717364413, 0.5296541538656578 ], "n": 100, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.4491" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "derived": true, "warm": "hms", "other": "lucene", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "hms", "value": 0.8203333333333334, "ci": [ 0.7538333333333334, 0.8826666666666667 ], "n": 100, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.8203" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "sqlite-fts5", "value": 0.8391666666666667, "ci": [ 0.7718333333333334, 0.8998333333333334 ], "n": 100, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.8392" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "lucene", "value": 0.8236666666666665, "ci": [ 0.7573333333333333, 0.8858333333333333 ], "n": 100, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.8237" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "tantivy", "value": 0.8203333333333334, "ci": [ 0.7538333333333334, 0.8826666666666667 ], "n": 100, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.8203" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "hms", "value": 12628.515668262533, "ci": [ 11050.469816249473, 12771.509520904918 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "12,629" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "sqlite-fts5", "value": 62.59960277572298, "ci": [ 53.23821643099081, 63.0343365952096 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "63" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "lucene", "value": 1338.3452757897326, "ci": [ 1154.2618967840313, 1367.5202080512422 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "1,338" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "tantivy", "value": 4849.943241114249, "ci": [ 4503.816556669204, 5298.197945125717 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "4,850" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "hms", "value": 75.674, "ci": [ 74.161, 82.426 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "76" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "sqlite-fts5", "value": 12431.824, "ci": [ 12368.788, 14777.976 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "12,432" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "lucene", "value": 688.722, "ci": [ 663.013, 819.938 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "689" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "tantivy", "value": 190.226, "ci": [ 179.244, 211.096 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "190" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "hms", "value": 144.633, "ci": [ 142.803, 174.308 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "145" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "sqlite-fts5", "value": 61156.511, "ci": [ 59640.913, 66734.402 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "61,157" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "lucene", "value": 1365.833, "ci": [ 1346.877, 1477.334 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "1,366" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "tantivy", "value": 396.206, "ci": [ 336.25, 449.623 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "396" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "hms", "value": 1.213714217, "ci": [ 1.196267904, 1.2146892440000001 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "1.21" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "sqlite-fts5", "value": 0.8815983789991151, "ci": [ 0.8279929869986518, 0.9001539719993161 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.88" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "lucene", "value": 2.25939896, "ci": [ 2.248518259, 2.347525725 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "2.26" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "tantivy", "value": 1.7403191430003062, "ci": [ 1.734949711999434, 1.7461702269993111 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "1.74" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "hms", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "sqlite-fts5", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "lucene", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "tantivy", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "hms", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].peak_rss_bytes", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "sqlite-fts5", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].peak_rss_bytes", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "lucene", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].peak_rss_bytes", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "tantivy", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].peak_rss_bytes", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "hms", "value": 9.797192573547363, "ci": [ 9.797192573547363, 9.797192573547363 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "9.8" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "sqlite-fts5", "value": 19.25390625, "ci": [ 19.25390625, 19.25390625 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "19.3" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "lucene", "value": 10.313986778259277, "ci": [ 10.313986778259277, 10.313986778259277 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "10.3" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "tantivy", "value": 13.706984519958496, "ci": [ 13.706984519958496, 13.706984519958496 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "13.7" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "derived": true, "warm": "hms", "other": "lucene", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "hms", "value": 0.301695743, "ci": [ 0.287584375, 0.317913052 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.30" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "sqlite-fts5", "value": 0.035247877, "ci": [ 0.031886552, 0.038362907 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.04" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "lucene", "value": 0.275996363, "ci": [ 0.271379881, 0.316352287 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.28" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "tantivy", "value": 0.022639198, "ci": [ 0.020162576, 0.022888295 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.02" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "hms", "value": 4.456856650000001, "ci": [ 4.456856650000001, 4.456856650000001 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "4.457" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "sqlite-fts5", "value": 11.852811349999683, "ci": [ 11.852811349999683, 11.852811349999683 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "11.853" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "lucene", "value": 38.05683817, "ci": [ 38.05683817, 38.05683817 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "38.057" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "tantivy", "value": 93.66708319999816, "ci": [ 93.66708319999816, 93.66708319999816 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "93.667" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "hms", "value": 3.3314022100000003, "ci": [ 3.3314022100000003, 3.3314022100000003 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "3.331" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "sqlite-fts5", "value": 10.455956980003975, "ci": [ 10.455956980003975, 10.455956980003975 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "10.456" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "lucene", "value": 17.638506319999998, "ci": [ 17.638506319999998, 17.638506319999998 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "17.639" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "tantivy", "value": 52.88391140000385, "ci": [ 52.88391140000385, 52.88391140000385 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "52.884" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "hms", "value": 2.4908218383789062, "ci": [ 2.4908218383789062, 2.4908218383789062 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "2.5" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "sqlite-fts5", "value": 0.0, "ci": [ 0.0, 0.0 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "0.0" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "lucene", "value": 5.715169906616211, "ci": [ 5.715169906616211, 5.715169906616211 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "5.7" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "tantivy", "value": 13.801004409790039, "ci": [ 13.801004409790039, 13.801004409790039 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "13.8" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "hms", "value": 92, "ci": [ 92, 92 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install._crate_dependencies", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "92" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "sqlite-fts5", "value": 0, "ci": [ 0, 0 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "0" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "lucene", "value": 2, "ci": [ 2, 2 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "2" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "tantivy", "value": 1, "ci": [ 1, 1 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "1" }, { "section": "memory (4,559 documents, 100 queries)", "file": "benchmarks/results/competitive/standard-text-memory-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "hms", "value": 0.29158226275481797, "ci": [ 0.25912963977629083, 0.32356876108307125 ], "n": 324, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.2916" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "sqlite-fts5", "value": 0.29140981016338025, "ci": [ 0.2580388719165204, 0.32356280551113314 ], "n": 324, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.2914" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "lucene", "value": 0.28972271928920523, "ci": [ 0.2558415031346977, 0.321223315571466 ], "n": 324, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.2897" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "tantivy", "value": 0.29006819680247453, "ci": [ 0.2567207176285045, 0.3220969741370192 ], "n": 324, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.2901" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "hms", "value": 0.22742500506264265, "ci": [ 0.19875352037951716, 0.25456973325714316 ], "n": 324, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.2274" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "sqlite-fts5", "value": 0.22713744545849535, "ci": [ 0.19870144179042648, 0.2541558791606999 ], "n": 324, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.2271" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "lucene", "value": 0.22757113786424402, "ci": [ 0.19879553549983092, 0.25449097683254396 ], "n": 324, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.2276" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "tantivy", "value": 0.22689859592232703, "ci": [ 0.1983430475484094, 0.25373280688143784 ], "n": 324, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.2269" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "derived": true, "warm": "hms", "other": "lucene", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "hms", "value": 40784.21499826919, "ci": [ 37478.85741307508, 41541.912712748446 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "40,784" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "sqlite-fts5", "value": 1415.6626698169298, "ci": [ 1359.7652789962624, 1453.5804663585916 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "1,416" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "lucene", "value": 4114.5853605973725, "ci": [ 3918.9511551362534, 4517.162897691404 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "4,115" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "tantivy", "value": 15413.471364458048, "ci": [ 14543.941714345583, 15804.180352038118 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "15,413" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "hms", "value": 25.791, "ci": [ 25.225, 27.906 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "26" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "sqlite-fts5", "value": 254.805, "ci": [ 247.493, 256.333 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "255" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "lucene", "value": 283.067, "ci": [ 263.002, 297.919 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "283" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "tantivy", "value": 70.693, "ci": [ 68.494, 75.971 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "71" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "hms", "value": 57.175, "ci": [ 53.701, 64.058 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "57" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "sqlite-fts5", "value": 2701.062, "ci": [ 2635.155, 2875.719 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "2,701" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "lucene", "value": 539.166, "ci": [ 471.91, 567.596 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "539" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "tantivy", "value": 149.59, "ci": [ 140.325, 156.595 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "150" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "hms", "value": 0.230839372, "ci": [ 0.23019802, 0.232152894 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.23" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "sqlite-fts5", "value": 0.13541231199997128, "ci": [ 0.1334877620010957, 0.14226272999985667 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.14" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "lucene", "value": 0.619037672, "ci": [ 0.614903845, 0.621166554 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.62" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "tantivy", "value": 0.2790714090006077, "ci": [ 0.27693536700098775, 0.2895067500012374 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.28" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "hms", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "sqlite-fts5", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "lucene", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "tantivy", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "hms", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].peak_rss_bytes", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "sqlite-fts5", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].peak_rss_bytes", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "lucene", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].peak_rss_bytes", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "tantivy", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].peak_rss_bytes", "validity": "invalid", "validity_reason": "Linux child inherited the orchestrator high-water mark; measured before the isolated-spawner fix", "uncertainty": "sampled", "text": "n/m" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "hms", "value": 2.0628175735473633, "ci": [ 2.0628175735473633, 2.0628175735473633 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "2.1" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "sqlite-fts5", "value": 4.0234375, "ci": [ 4.0234375, 4.0234375 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "4.0" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "lucene", "value": 1.4865550994873047, "ci": [ 1.4865550994873047, 1.4865550994873047 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "1.5" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "tantivy", "value": 1.9688215255737305, "ci": [ 1.9688215255737305, 1.9688215255737305 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "2.0" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "derived": true, "warm": "hms", "other": "lucene", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "hms", "value": 0.08889909, "ci": [ 0.083722021, 0.097128282 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.09" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "sqlite-fts5", "value": 0.021537669, "ci": [ 0.020218705, 0.02514792 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.02" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "lucene", "value": 0.283127535, "ci": [ 0.272270907, 0.30104404 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.28" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "tantivy", "value": 0.020183307, "ci": [ 0.019923964, 0.025963645 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.02" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "hms", "value": 2.03304533, "ci": [ 2.03304533, 2.03304533 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "2.033" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "sqlite-fts5", "value": 9.737997880001785, "ci": [ 9.737997880001785, 9.737997880001785 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "9.738" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "lucene", "value": 30.87565001, "ci": [ 30.87565001, 30.87565001 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "30.876" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "tantivy", "value": 84.13805727999716, "ci": [ 84.13805727999716, 84.13805727999716 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "84.138" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "hms", "value": 1.7135118999999999, "ci": [ 1.7135118999999999, 1.7135118999999999 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "1.714" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "sqlite-fts5", "value": 9.43153046999214, "ci": [ 9.43153046999214, 9.43153046999214 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "9.432" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "lucene", "value": 14.462594180000002, "ci": [ 14.462594180000002, 14.462594180000002 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "14.463" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "tantivy", "value": 42.87158723000175, "ci": [ 42.87158723000175, 42.87158723000175 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "42.872" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "hms", "value": 2.4908218383789062, "ci": [ 2.4908218383789062, 2.4908218383789062 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "2.5" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "sqlite-fts5", "value": 0.0, "ci": [ 0.0, 0.0 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "0.0" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "lucene", "value": 5.715169906616211, "ci": [ 5.715169906616211, 5.715169906616211 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "5.7" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "tantivy", "value": 13.801004409790039, "ci": [ 13.801004409790039, 13.801004409790039 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "13.8" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "hms", "value": 92, "ci": [ 92, 92 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install._crate_dependencies", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "92" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "sqlite-fts5", "value": 0, "ci": [ 0, 0 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "0" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "lucene", "value": 2, "ci": [ 2, 2 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "2" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "tantivy", "value": 1, "ci": [ 1, 1 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "1" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/standard-text-beir-nfcorpus-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "hms", "value": 0.29158226275481797, "ci": [ 0.2389921791965676, 0.33934188094051343 ], "n": 324, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.2916" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "hms-eager", "value": 0.29158226275481797, "ci": [ 0.2389921791965676, 0.33934188094051343 ], "n": 324, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.2916" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "sqlite-fts5", "value": 0.29140981016338025, "ci": [ 0.23913017727550598, 0.3404583607962663 ], "n": 324, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.2914" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "lucene", "value": 0.28972271928920523, "ci": [ 0.23724687603899755, 0.3387393508400861 ], "n": 324, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.2897" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "tantivy", "value": 0.29006819680247453, "ci": [ 0.23766263377150765, 0.3373278424145376 ], "n": 324, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.2901" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "whoosh", "value": 0.2873130912847506, "ci": [ 0.23469301865019682, 0.33341568767350244 ], "n": 324, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.2873" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "bm25s", "value": 0.287785039883203, "ci": [ 0.2346829683464342, 0.334993878551926 ], "n": 324, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.2878" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 1.0005917871856855, "factor_infinite": false, "ci": [ 0.7019718318493078, 1.419067575689336 ], "verdict": "TIE", "text": "1.00x" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "hms", "value": 0.22742500506264265, "ci": [ 0.18530820446651203, 0.2699444227525327 ], "n": 324, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.2274" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "hms-eager", "value": 0.22742500506264265, "ci": [ 0.18530820446651203, 0.2699444227525327 ], "n": 324, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.2274" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "sqlite-fts5", "value": 0.22713744545849535, "ci": [ 0.18639467777258142, 0.2676392828738551 ], "n": 324, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.2271" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "lucene", "value": 0.22757113786424402, "ci": [ 0.18623200296625053, 0.2686162528083917 ], "n": 324, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.2276" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "tantivy", "value": 0.22689859592232703, "ci": [ 0.18437031563892306, 0.26936672640729253 ], "n": 324, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.2269" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "whoosh", "value": 0.2251819508302511, "ci": [ 0.18545926318943576, 0.26578182472231626 ], "n": 324, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.2252" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "bm25s", "value": 0.22715065794318523, "ci": [ 0.18544371473099353, 0.2681726673825259 ], "n": 324, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.2272" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "derived": true, "warm": "hms", "other": "lucene", "factor": 0.9993578588085782, "factor_infinite": false, "ci": [ 0.6898622199107787, 1.4495060916111866 ], "verdict": "TIE", "text": "1.00x" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "hms", "value": 10571.528830237587, "ci": [ 8628.496028907804, 11081.766471882958 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "10,572" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "hms-eager", "value": 10809.300798015202, "ci": [ 10166.927455677765, 11110.307556974018 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "10,809" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "sqlite-fts5", "value": 486.387220762379, "ci": [ 477.8881513725941, 493.73192764948027 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "486" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "lucene", "value": 3022.570796310708, "ci": [ 2952.7388448123756, 3069.127289581983 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "3,023" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "tantivy", "value": 7151.092941690469, "ci": [ 6813.435354293591, 7379.63650041182 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "7,151" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "whoosh", "value": 131.37953514213092, "ci": [ 130.39154395383645, 132.51555600623206 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "131" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "bm25s", "value": 1232.7320529292738, "ci": [ 1177.8034777243208, 1288.3462870911721 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "1,233" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "derived": true, "warm": "hms", "other": "tantivy", "factor": 1.478309527849396, "factor_infinite": false, "ci": [ 1.1692304937277456, 1.6264580047566772 ], "verdict": "WIN", "text": "1.48x" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "hms", "value": 59.5155, "ci": [ 56.209, 78.001 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "60" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "hms-eager", "value": 57.0405, "ci": [ 55.146, 62.61 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "57" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "sqlite-fts5", "value": 774.52, "ci": [ 758.193, 784.624 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "775" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "lucene", "value": 405.8085, "ci": [ 395.864, 410.793 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "406" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "tantivy", "value": 156.923, "ci": [ 155.62, 164.648 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "157" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "whoosh", "value": 2968.8845, "ci": [ 2931.569, 3050.117 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "2,969" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "bm25s", "value": 798.5985000000001, "ci": [ 775.367, 843.713 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "799" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "derived": true, "warm": "hms", "other": "tantivy", "factor": 2.636674479757374, "factor_infinite": false, "ci": [ 1.9951026268893988, 2.9292106246330656 ], "verdict": "WIN", "text": "2.64x" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "hms", "value": 279.0985, "ci": [ 262.466, 334.495 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "279" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "hms-eager", "value": 274.59000000000003, "ci": [ 261.846, 303.196 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "275" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "sqlite-fts5", "value": 6962.541499999999, "ci": [ 6752.032, 7127.956 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "6,963" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "lucene", "value": 703.3555, "ci": [ 679.512, 740.936 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "703" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "tantivy", "value": 293.677, "ci": [ 280.491, 317.212 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "294" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "whoosh", "value": 33136.378, "ci": [ 32623.342, 33851.841 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "33,136" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "bm25s", "value": 1082.0439999999999, "ci": [ 1032.613, 1189.952 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "1,082" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "derived": true, "warm": "hms", "other": "tantivy", "factor": 1.052234247048981, "factor_infinite": false, "ci": [ 0.83855065098133, 1.2085832069677596 ], "verdict": "TIE", "text": "1.05x" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "hms", "value": 0.5921563255, "ci": [ 0.58976631, 0.594546341 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.59" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "hms-eager", "value": 2.179494596, "ci": [ 2.170529804, 2.188459388 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "2.18" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "sqlite-fts5", "value": 0.285780587999966, "ci": [ 0.2805567420000443, 0.2910044339998876 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.29" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "lucene", "value": 0.966967628, "ci": [ 0.965895586, 0.96803967 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.97" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "tantivy", "value": 0.485183013999972, "ci": [ 0.4764130700000351, 0.4939529579999089 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.49" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "whoosh", "value": 7.334148889499943, "ci": [ 7.328755410999975, 7.339542367999911 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "7.33" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "bm25s", "value": 0.8843612315000655, "ci": [ 0.8741987220000738, 0.8945237410000573 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.88" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 0.48261004010834635, "factor_infinite": false, "ci": [ 0.4718837248719092, 0.4934232916761346 ], "verdict": "LOSS", "text": "0.48x" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "hms", "value": 57.302734375, "ci": [ 56.78125, 57.82421875 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "57" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "hms-eager", "value": 86.779296875, "ci": [ 86.75390625, 86.8046875 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "87" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "sqlite-fts5", "value": 33.82421875, "ci": [ 33.7890625, 33.859375 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "34" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "lucene", "value": 454.845703125, "ci": [ 448.98828125, 460.703125 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "455" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "tantivy", "value": 77.69140625, "ci": [ 75.26171875, 80.12109375 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "78" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "whoosh", "value": 144.1953125, "ci": [ 143.53125, 144.859375 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "144" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "bm25s", "value": 91.8046875, "ci": [ 91.703125, 91.90625 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "92" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 0.5902723337537067, "factor_infinite": false, "ci": [ 0.5843410119570357, 0.5963126031920748 ], "verdict": "LOSS", "text": "0.59x" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "hms", "value": 56.791015625, "ci": [ 56.17578125, 57.99609375 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "57" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "hms-eager", "value": 58.37109375, "ci": [ 57.640625, 58.87890625 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "58" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "sqlite-fts5", "value": 29.9296875, "ci": [ 28.984375, 30.24609375 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "30" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "lucene", "value": 441.1484375, "ci": [ 435.97265625, 444.203125 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "441" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "tantivy", "value": 41.001953125, "ci": [ 39.96484375, 41.5078125 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "41" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "whoosh", "value": 69.625, "ci": [ 68.90625, 70.130859375 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "70" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "bm25s", "value": 70.138671875, "ci": [ 69.5078125, 70.66796875 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "70" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 0.5270144787976752, "factor_infinite": false, "ci": [ 0.49976426214049974, 0.5384187469577916 ], "verdict": "LOSS", "text": "0.53x" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "hms", "value": 2.0628175735473633, "ci": [ 2.0628175735473633, 2.0628175735473633 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "2.1" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "hms-eager", "value": 2.0628175735473633, "ci": [ 2.0628175735473633, 2.0628175735473633 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "2.1" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "sqlite-fts5", "value": 9.3046875, "ci": [ 9.3046875, 9.3046875 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "9.3" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "lucene", "value": 1.4865388870239258, "ci": [ 1.4865388870239258, 1.4865388870239258 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "1.5" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "tantivy", "value": 1.9688215255737305, "ci": [ 1.9688215255737305, 1.9688215255737305 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "2.0" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "whoosh", "value": 9.888919830322266, "ci": [ 9.888919830322266, 9.888919830322266 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "9.9" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "bm25s", "value": 3.3759756088256836, "ci": [ 3.3759756088256836, 3.3759756088256836 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "3.4" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "derived": true, "warm": "hms", "other": "lucene", "factor": 0.7206351672036472, "factor_infinite": false, "ci": null, "verdict": "LOSS", "text": "0.72x" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "hms", "value": 0.175434672, "ci": [ 0.170015857, 0.187209304 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.18" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "hms-eager", "value": 0.181669582, "ci": [ 0.176235174, 0.189533834 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.18" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "sqlite-fts5", "value": 0.068488589, "ci": [ 0.065624187, 0.06911327 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.07" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "lucene", "value": 0.4379882085, "ci": [ 0.418019085, 0.447336709 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.44" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "tantivy", "value": 0.068223135, "ci": [ 0.065778959, 0.072269295 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.07" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "whoosh", "value": 0.241962255, "ci": [ 0.231076263, 0.258533869 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.24" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "bm25s", "value": 0.20877574, "ci": [ 0.199165031, 0.219764162 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.21" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "derived": true, "warm": "hms", "other": "tantivy", "factor": 0.38888056860276743, "factor_infinite": false, "ci": [ 0.35136586480765936, 0.4250738506114756 ], "verdict": "LOSS", "text": "0.39x" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "hms", "value": 1.07312056, "ci": [ 1.07312056, 1.07312056 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "1.073" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "hms-eager", "value": 1.06455452, "ci": [ 1.06455452, 1.06455452 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "1.065" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "sqlite-fts5", "value": 3.140645449998374, "ci": [ 3.140645449998374, 3.140645449998374 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "3.141" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "lucene", "value": 18.21623659, "ci": [ 18.21623659, 18.21623659 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "18.216" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "tantivy", "value": 55.32423113999812, "ci": [ 55.32423113999812, 55.32423113999812 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "55.324" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "whoosh", "value": 53.212382009999146, "ci": [ 53.212382009999146, 53.212382009999146 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "53.212" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "bm25s", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.ops", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "n/m" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 2.9266473563775293, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "2.93x" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "hms", "value": 0.72923147, "ci": [ 0.72923147, 0.72923147 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "0.729" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "hms-eager", "value": 0.7055342400000001, "ci": [ 0.7055342400000001, 0.7055342400000001 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "0.706" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "sqlite-fts5", "value": 8.833210899999813, "ci": [ 8.833210899999813, 8.833210899999813 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "8.833" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "lucene", "value": 1.3494846699999998, "ci": [ 1.3494846699999998, 1.3494846699999998 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "1.349" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "tantivy", "value": 50.4753322800002, "ci": [ 50.4753322800002, 50.4753322800002 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "50.475" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "whoosh", "value": 6.140949739999542, "ci": [ 6.140949739999542, 6.140949739999542 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "6.141" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "bm25s", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.ops", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "n/m" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "derived": true, "warm": "hms", "other": "lucene", "factor": 1.8505573682934993, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "1.85x" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "hms", "value": 2.3764114379882812, "ci": [ 2.3764114379882812, 2.3764114379882812 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "2.4" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "hms-eager", "value": 2.3764114379882812, "ci": [ 2.3764114379882812, 2.3764114379882812 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "2.4" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "sqlite-fts5", "value": 0.0, "ci": [ 0.0, 0.0 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.install", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "0.0" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "lucene", "value": 5.715169906616211, "ci": [ 5.715169906616211, 5.715169906616211 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.install", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "5.7" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "tantivy", "value": 13.801004409790039, "ci": [ 13.801004409790039, 13.801004409790039 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.install", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "13.8" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "whoosh", "value": 1.9728384017944336, "ci": [ 1.9728384017944336, 1.9728384017944336 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.install", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "2.0" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "bm25s", "value": 56.54034996032715, "ci": [ 56.54034996032715, 56.54034996032715 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.install", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "56.5" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "derived": true, "warm": "hms", "other": "whoosh", "factor": 0.8301754360619107, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "0.83x" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "hms", "value": 92, "ci": [ 92, 92 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms.install._crate_dependencies", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "92" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "hms-eager", "value": 92, "ci": [ 92, 92 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.hms-eager.install._crate_dependencies", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "92" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "sqlite-fts5", "value": 0, "ci": [ 0, 0 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "0" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "lucene", "value": 2, "ci": [ 2, 2 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.lucene.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "2" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "tantivy", "value": 1, "ci": [ 1, 1 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.tantivy.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "1" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "whoosh", "value": 2, "ci": [ 2, 2 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.whoosh.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "2" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "bm25s", "value": 3, "ci": [ 3, 3 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json#systems.bm25s.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "3" }, { "section": "beir:nfcorpus (3,633 documents, 324 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-nfcorpus-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "derived": true, "warm": "hms", "other": "tantivy", "factor": 0.010869565217391304, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "0.01x" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "hms", "value": 0.7026997341473034, "ci": [ 0.6767786729633418, 0.7322695862845088 ], "n": 809, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.7027" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "sqlite-fts5", "value": 0.7009278481912261, "ci": [ 0.6753581453218557, 0.7306336500865396 ], "n": 809, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.7009" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "lucene", "value": 0.7021782838562147, "ci": [ 0.6765209168126257, 0.7322968706728786 ], "n": 809, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.7022" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "system": "tantivy", "value": 0.701325671999324, "ci": [ 0.6755941449618927, 0.7308008746966628 ], "n": 809, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.7013" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "ndcg@10", "derived": true, "warm": "hms", "other": "lucene", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "hms", "value": 0.9264112072517512, "ci": [ 0.9080758137618459, 0.9446847960444994 ], "n": 809, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.9264" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "sqlite-fts5", "value": 0.9321796456530697, "ci": [ 0.9157601977750309, 0.9490111248454882 ], "n": 809, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.9322" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "lucene", "value": 0.9295014421096004, "ci": [ 0.9130613926658425, 0.9474660074165636 ], "n": 809, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.9295" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "system": "tantivy", "value": 0.9282653481664607, "ci": [ 0.9110630407911001, 0.9466007416563659 ], "n": 809, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[0].quality", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.9283" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "recall@100", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "hms", "value": 17781.668497694965, "ci": [ 16670.54615758954, 17880.6832834575 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "17,782" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "sqlite-fts5", "value": 242.16717008571558, "ci": [ 239.0382548446373, 243.66628093250253 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "242" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "lucene", "value": 2862.2097821881707, "ci": [ 2518.397806522205, 2929.936108270389 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "2,862" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "system": "tantivy", "value": 6450.044975709162, "ci": [ 6418.39423883029, 6507.677325740711 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "6,450" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "qps", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "hms", "value": 55.432, "ci": [ 54.937, 56.628 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "55" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "sqlite-fts5", "value": 3852.758, "ci": [ 3840.981, 3879.967 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "3,853" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "lucene", "value": 329.247, "ci": [ 319.601, 368.767 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "329" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "system": "tantivy", "value": 150.532, "ci": [ 150.181, 150.772 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "151" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p50_latency", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "hms", "value": 94.551, "ci": [ 81.733, 103.839 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "95" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "sqlite-fts5", "value": 9342.801, "ci": [ 9318.986, 9672.006 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "9,343" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "lucene", "value": 789.441, "ci": [ 717.179, 865.042 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "789" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "system": "tantivy", "value": 287.282, "ci": [ 242.068, 319.241 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].passes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "287" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "p99_latency", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "hms", "value": 0.312045625, "ci": [ 0.310915802, 0.317366839 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.31" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "sqlite-fts5", "value": 0.19639155000004394, "ci": [ 0.1934057339985884, 0.21986437700070383 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.20" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "lucene", "value": 0.712891089, "ci": [ 0.682538507, 0.789371421 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.71" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "system": "tantivy", "value": 0.36752783200063277, "ci": [ 0.3633899329997803, 0.3676003689997742 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.37" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_seconds", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "hms", "value": 73.18359375, "ci": [ 73.18359375, 73.18359375 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "73" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "sqlite-fts5", "value": 73.18359375, "ci": [ 73.18359375, 73.18359375 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "73" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "lucene", "value": 162.078125, "ci": [ 158.83984375, 167.75 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "162" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "system": "tantivy", "value": 73.18359375, "ci": [ 73.18359375, 73.18359375 ], "n": 3, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "73" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "build_peak_rss_mib", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "hms", "value": 195.8359375, "ci": [ 143.99609375, 329.3828125 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].peak_rss_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "196" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "sqlite-fts5", "value": 218.0078125, "ci": [ 84.171875, 351.5546875 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].peak_rss_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "218" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "lucene", "value": 410.25390625, "ci": [ 403.73046875, 426.3046875 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].peak_rss_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "410" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "system": "tantivy", "value": 173.6640625, "ci": [ 84.859375, 396.0703125 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].peak_rss_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "174" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "serving_rss_mib", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "hms", "value": 2.9104738235473633, "ci": [ 2.9104738235473633, 2.9104738235473633 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "2.9" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "sqlite-fts5", "value": 5.87109375, "ci": [ 5.87109375, 5.87109375 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "5.9" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "lucene", "value": 1.9866275787353516, "ci": [ 1.9866275787353516, 1.9866275787353516 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "2.0" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "system": "tantivy", "value": 2.6494836807250977, "ci": [ 2.6494836807250977, 2.6494836807250977 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.build_runs[0].disk_bytes", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "2.6" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "disk_mib", "derived": true, "warm": "hms", "other": "lucene", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "hms", "value": 0.117377458, "ci": [ 0.11571311, 0.119997146 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.12" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "sqlite-fts5", "value": 0.026888259, "ci": [ 0.023155602, 0.029468671 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.03" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "lucene", "value": 0.291827871, "ci": [ 0.28430985, 2.911798323 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.29" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "system": "tantivy", "value": 0.024623582, "ci": [ 0.023976055, 0.025707532 ], "n": 5, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.serve_rounds[*].cold_start_s", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "sampled", "text": "0.02" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "cold_start_s", "derived": true, "warm": "hms", "other": "tantivy", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "hms", "value": 2.00371566, "ci": [ 2.00371566, 2.00371566 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "2.004" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "sqlite-fts5", "value": 9.131574970015208, "ci": [ 9.131574970015208, 9.131574970015208 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "9.132" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "lucene", "value": 30.77421369, "ci": [ 30.77421369, 30.77421369 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "30.774" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "system": "tantivy", "value": 86.06415542000832, "ci": [ 86.06415542000832, 86.06415542000832 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.replace", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "86.064" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "replace_ms_per_op", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "hms", "value": 1.73026945, "ci": [ 1.73026945, 1.73026945 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "1.730" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "sqlite-fts5", "value": 8.99912264001614, "ci": [ 8.99912264001614, 8.99912264001614 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "8.999" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "lucene", "value": 14.38683205, "ci": [ 14.38683205, 14.38683205 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "14.387" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "system": "tantivy", "value": 44.8263032900104, "ci": [ 44.8263032900104, 44.8263032900104 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.ops.delete", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "44.826" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "delete_ms_per_op", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "hms", "value": 2.4908218383789062, "ci": [ 2.4908218383789062, 2.4908218383789062 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "2.5" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "sqlite-fts5", "value": 0.0, "ci": [ 0.0, 0.0 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "0.0" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "lucene", "value": 5.715169906616211, "ci": [ 5.715169906616211, 5.715169906616211 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "5.7" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "system": "tantivy", "value": 13.801004409790039, "ci": [ 13.801004409790039, 13.801004409790039 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "deterministic", "text": "13.8" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_mib", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "hms", "value": 92, "ci": [ 92, 92 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.hms.install._crate_dependencies", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "92" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "sqlite-fts5", "value": 0, "ci": [ 0, 0 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.sqlite-fts5.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "0" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "lucene", "value": 2, "ci": [ 2, 2 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.lucene.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "2" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "system": "tantivy", "value": 1, "ci": [ 1, 1 ], "n": 1, "source": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json#systems.tantivy.install", "validity": "unverified", "validity_reason": "source revision is not pinned", "uncertainty": "unreplicated", "text": "1" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/standard-text-beir-scifact-linux-x86_64-intel-ultra-9-185h-22c-27g.json", "hardware_class": "linux-x86_64-intel-ultra-9-185h-22c-27g", "metric": "install_distributions", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": null, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "n/v" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "hms", "value": 0.7026997341473034, "ci": [ 0.6664189500472893, 0.7419550353136701 ], "n": 809, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.7027" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "hms-eager", "value": 0.7026997341473034, "ci": [ 0.6664189500472893, 0.7419550353136701 ], "n": 809, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.7027" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "sqlite-fts5", "value": 0.7009278481912261, "ci": [ 0.6643714062414602, 0.7404343587518659 ], "n": 809, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.7009" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "lucene", "value": 0.7021782838562147, "ci": [ 0.6659037403180936, 0.741114931390745 ], "n": 809, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.7022" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "tantivy", "value": 0.701325671999324, "ci": [ 0.6640624257389993, 0.7411381074045522 ], "n": 809, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.7013" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "whoosh", "value": 0.6907626470247572, "ci": [ 0.6518433173669151, 0.7289895253481598 ], "n": 809, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.6908" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "system": "bm25s", "value": 0.700081542866468, "ci": [ 0.6619669702121025, 0.737860727292892 ], "n": 809, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.7001" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "ndcg@10", "derived": true, "warm": "hms", "other": "lucene", "factor": 1.0007426180830101, "factor_infinite": false, "ci": [ 0.8992113393218454, 1.114207640520607 ], "verdict": "TIE", "text": "1.00x" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "hms", "value": 0.9264112072517512, "ci": [ 0.8996909765142151, 0.9501030078285949 ], "n": 809, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.9264" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "hms-eager", "value": 0.9264112072517512, "ci": [ 0.8996909765142151, 0.9501030078285949 ], "n": 809, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.9264" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "sqlite-fts5", "value": 0.9321796456530697, "ci": [ 0.9083848372476309, 0.9558096415327565 ], "n": 809, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.9322" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "lucene", "value": 0.9295014421096004, "ci": [ 0.9023691800576844, 0.9522661722290894 ], "n": 809, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.9295" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "tantivy", "value": 0.9282653481664607, "ci": [ 0.9024103831891224, 0.9524103831891224 ], "n": 809, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.9283" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "whoosh", "value": 0.9185826122785331, "ci": [ 0.888710341985991, 0.9443757725587144 ], "n": 809, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.9186" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "system": "bm25s", "value": 0.9282653481664607, "ci": [ 0.9009064688916358, 0.9501030078285949 ], "n": 809, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[0].quality", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.9283" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "recall@100", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 0.9938118811881188, "factor_infinite": false, "ci": [ 0.9412867765923053, 1.0459256571337854 ], "verdict": "TIE", "text": "0.99x" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "hms", "value": 2732.645695786485, "ci": [ 2632.5697276850024, 2808.922891816741 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "2,733" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "hms-eager", "value": 2701.8609694681927, "ci": [ 2611.901411279101, 2782.887516243859 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "2,702" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "sqlite-fts5", "value": 99.47137943955144, "ci": [ 97.34158790255437, 102.312572751274 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "99" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "lucene", "value": 1457.3055473838826, "ci": [ 1421.054644223512, 1540.472346295411 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "1,457" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "tantivy", "value": 2964.8401183361775, "ci": [ 2853.0639279227885, 3029.1496653076892 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "2,965" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "whoosh", "value": 23.438295567347772, "ci": [ 22.35560006516289, 24.001002269987797 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "23" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "system": "bm25s", "value": 1126.786683480005, "ci": [ 1064.4720426825868, 1193.315123610056 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "1,127" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "qps", "derived": true, "warm": "hms", "other": "tantivy", "factor": 0.9216839987041201, "factor_infinite": false, "ci": [ 0.8690787906042919, 0.9845285499304655 ], "verdict": "LOSS", "text": "0.92x" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "hms", "value": 372.6545, "ci": [ 361.298, 385.574 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "373" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "hms-eager", "value": 376.171, "ci": [ 367.069, 389.382 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "376" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "sqlite-fts5", "value": 9931.3925, "ci": [ 9670.728, 10183.449 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "9,931" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "lucene", "value": 641.6034999999999, "ci": [ 618.825, 655.475 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "642" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "tantivy", "value": 334.0, "ci": [ 327.7325, 346.469 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "334" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "whoosh", "value": 36022.854999999996, "ci": [ 35006.087, 37407.892 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "36,023" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "system": "bm25s", "value": 870.7950000000001, "ci": [ 822.502, 916.248 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "871" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p50_latency", "derived": true, "warm": "hms", "other": "tantivy", "factor": 0.8962725527264531, "factor_infinite": false, "ci": [ 0.8499859949062956, 0.958956318606801 ], "verdict": "LOSS", "text": "0.90x" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "hms", "value": 518.2855, "ci": [ 510.285, 544.661 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "518" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "hms-eager", "value": 530.163, "ci": [ 509.442, 547.797 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "530" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "sqlite-fts5", "value": 17709.493499999997, "ci": [ 17362.939, 18452.92 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "17,709" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "lucene", "value": 1127.601, "ci": [ 1061.158, 1197.361 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "1,128" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "tantivy", "value": 508.30050000000006, "ci": [ 473.705, 609.637 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "508" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "whoosh", "value": 128595.2095, "ci": [ 127226.219, 137014.661 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "128,595" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "system": "bm25s", "value": 1258.5345000000002, "ci": [ 1168.826, 1339.244 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].passes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "1,259" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "p99_latency", "derived": true, "warm": "hms", "other": "tantivy", "factor": 0.9807345565330308, "factor_infinite": false, "ci": [ 0.8697244708176279, 1.1946990407321396 ], "verdict": "TIE", "text": "0.98x" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "hms", "value": 1.4185485660000001, "ci": [ 1.327840441, 1.509256691 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "1.42" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "hms-eager", "value": 3.6037260925, "ci": [ 3.56142862, 3.646023565 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "3.60" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "sqlite-fts5", "value": 0.42722204650000606, "ci": [ 0.4247947750000094, 0.4296493180000027 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.43" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "lucene", "value": 1.5220980705, "ci": [ 1.384241498, 1.659954643 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "1.52" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "tantivy", "value": 0.7557944099999929, "ci": [ 0.7396925189999877, 0.7718963009999982 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.76" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "whoosh", "value": 11.381427306000006, "ci": [ 11.27367346600002, 11.489181145999993 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "11.38" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "system": "bm25s", "value": 1.4012112320000085, "ci": [ 1.306480692000008, 1.495941772000009 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "1.40" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_seconds", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 0.3011684314092113, "factor_infinite": false, "ci": [ 0.28145959367491674, 0.3235699898373578 ], "verdict": "LOSS", "text": "0.30x" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "hms", "value": 65.451171875, "ci": [ 65.1484375, 65.75390625 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "65" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "hms-eager", "value": 95.69140625, "ci": [ 95.4140625, 95.96875 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "96" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "sqlite-fts5", "value": 39.20703125, "ci": [ 38.671875, 39.7421875 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "39" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "lucene", "value": 504.421875, "ci": [ 502.16796875, 506.67578125 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "504" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "tantivy", "value": 90.45703125, "ci": [ 84.25, 96.6640625 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "90" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "whoosh", "value": 178.310546875, "ci": [ 178.03125, 178.58984375 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "178" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "system": "bm25s", "value": 104.4609375, "ci": [ 104.28125, 104.640625 ], "n": 2, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "104" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "build_peak_rss_mib", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 0.5990271851033989, "factor_infinite": false, "ci": [ 0.5881304580288719, 0.6100251828756446 ], "verdict": "LOSS", "text": "0.60x" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "hms", "value": 159.7421875, "ci": [ 158.28515625, 160.828125 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "160" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "hms-eager", "value": 160.318359375, "ci": [ 158.873046875, 161.94140625 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "160" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "sqlite-fts5", "value": 54.490234375, "ci": [ 53.89453125, 54.88671875 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "54" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "lucene", "value": 951.931640625, "ci": [ 935.44140625, 971.515625 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "952" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "tantivy", "value": 65.037109375, "ci": [ 64.04296875, 65.734375 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "65" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "whoosh", "value": 101.42578125, "ci": [ 100.703125, 102.3984375 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "101" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "system": "bm25s", "value": 95.208984375, "ci": [ 94.09375, 96.24609375 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].peak_rss_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "95" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "serving_rss_mib", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 0.34111361079865016, "factor_infinite": false, "ci": [ 0.3351063829787234, 0.346758470916315 ], "verdict": "LOSS", "text": "0.34x" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "hms", "value": 2.910472869873047, "ci": [ 2.910472869873047, 2.910472869873047 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "2.9" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "hms-eager", "value": 2.906566619873047, "ci": [ 2.906566619873047, 2.906566619873047 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "2.9" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "sqlite-fts5", "value": 12.4375, "ci": [ 12.4375, 12.4375 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "12.4" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "lucene", "value": 1.9866113662719727, "ci": [ 1.9866113662719727, 1.9866113662719727 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "2.0" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "tantivy", "value": 2.6494836807250977, "ci": [ 2.6494836807250977, 2.6494836807250977 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "2.6" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "whoosh", "value": 13.159749031066895, "ci": [ 13.159749031066895, 13.159749031066895 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "13.2" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "system": "bm25s", "value": 4.487649917602539, "ci": [ 4.487649917602539, 4.487649917602539 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.build_runs[0].disk_bytes", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "4.5" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "disk_mib", "derived": true, "warm": "hms", "other": "lucene", "factor": 0.6825734013313883, "factor_infinite": false, "ci": null, "verdict": "LOSS", "text": "0.68x" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "hms", "value": 0.23930724650000001, "ci": [ 0.225467531, 0.268876353 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.24" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "hms-eager", "value": 0.2479915995, "ci": [ 0.241085405, 0.253164824 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.25" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "sqlite-fts5", "value": 0.0766115495, "ci": [ 0.073073611, 0.08288524 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.08" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "lucene", "value": 0.4577830215, "ci": [ 0.446489742, 0.514178286 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.46" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "tantivy", "value": 0.080238569, "ci": [ 0.075746264, 0.085537997 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.08" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "whoosh", "value": 0.2392416195, "ci": [ 0.226745583, 0.252238708 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.24" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "system": "bm25s", "value": 0.23249816099999998, "ci": [ 0.229607241, 0.2436112325 ], "n": 10, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.serve_rounds[*].cold_start_s", "validity": "valid", "validity_reason": "", "uncertainty": "sampled", "text": "0.23" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "cold_start_s", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 0.32013886173731054, "factor_infinite": false, "ci": [ 0.2717740336205765, 0.36761497157655043 ], "verdict": "LOSS", "text": "0.32x" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "hms", "value": 1.01634077, "ci": [ 1.01634077, 1.01634077 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "1.016" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "hms-eager", "value": 0.9228250499999999, "ci": [ 0.9228250499999999, 0.9228250499999999 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "0.923" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "sqlite-fts5", "value": 3.199492190001365, "ci": [ 3.199492190001365, 3.199492190001365 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "3.199" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "lucene", "value": 16.97920893, "ci": [ 16.97920893, 16.97920893 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "16.979" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "tantivy", "value": 73.3689347099994, "ci": [ 73.3689347099994, 73.3689347099994 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "73.369" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "whoosh", "value": 48.78160093999895, "ci": [ 48.78160093999895, 48.78160093999895 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.ops.replace", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "48.782" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "system": "bm25s", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.ops", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "n/m" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "replace_ms_per_op", "derived": true, "warm": "hms", "other": "sqlite-fts5", "factor": 3.14805061888973, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "3.15x" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "hms", "value": 0.79275055, "ci": [ 0.79275055, 0.79275055 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "0.793" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "hms-eager", "value": 0.71397007, "ci": [ 0.71397007, 0.71397007 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "0.714" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "sqlite-fts5", "value": 6.279815929999586, "ci": [ 6.279815929999586, 6.279815929999586 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "6.280" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "lucene", "value": 1.26921317, "ci": [ 1.26921317, 1.26921317 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "1.269" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "tantivy", "value": 63.29283183000144, "ci": [ 63.29283183000144, 63.29283183000144 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "63.293" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "whoosh", "value": 6.582569769998372, "ci": [ 6.582569769998372, 6.582569769998372 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.ops.delete", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "6.583" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "system": "bm25s", "value": null, "ci": null, "n": 0, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.ops", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "n/m" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "delete_ms_per_op", "derived": true, "warm": "hms", "other": "lucene", "factor": 1.601024647665019, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "1.60x" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "hms", "value": 2.3764114379882812, "ci": [ 2.3764114379882812, 2.3764114379882812 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "2.4" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "hms-eager", "value": 2.3764114379882812, "ci": [ 2.3764114379882812, 2.3764114379882812 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "2.4" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "sqlite-fts5", "value": 0.0, "ci": [ 0.0, 0.0 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.install", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "0.0" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "lucene", "value": 5.715169906616211, "ci": [ 5.715169906616211, 5.715169906616211 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.install", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "5.7" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "tantivy", "value": 13.801004409790039, "ci": [ 13.801004409790039, 13.801004409790039 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.install", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "13.8" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "whoosh", "value": 1.9728384017944336, "ci": [ 1.9728384017944336, 1.9728384017944336 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.install", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "2.0" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "system": "bm25s", "value": 56.54034996032715, "ci": [ 56.54034996032715, 56.54034996032715 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.install", "validity": "valid", "validity_reason": "", "uncertainty": "deterministic", "text": "56.5" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_mib", "derived": true, "warm": "hms", "other": "whoosh", "factor": 0.8301754360619107, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "0.83x" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "hms", "value": 92, "ci": [ 92, 92 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms.install._crate_dependencies", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "92" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "hms-eager", "value": 92, "ci": [ 92, 92 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.hms-eager.install._crate_dependencies", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "92" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "sqlite-fts5", "value": 0, "ci": [ 0, 0 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.sqlite-fts5.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "0" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "lucene", "value": 2, "ci": [ 2, 2 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.lucene.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "2" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "tantivy", "value": 1, "ci": [ 1, 1 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.tantivy.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "1" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "whoosh", "value": 2, "ci": [ 2, 2 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.whoosh.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "2" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "system": "bm25s", "value": 3, "ci": [ 3, 3 ], "n": 1, "source": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json#systems.bm25s.install", "validity": "valid", "validity_reason": "", "uncertainty": "unreplicated", "text": "3" }, { "section": "beir:scifact (5,183 documents, 809 queries)", "file": "benchmarks/results/competitive/heavy-text-beir-scifact-modal-x86_64-8vcpu-32g.json", "hardware_class": "modal-x86_64-8vcpu-32g", "metric": "install_distributions", "derived": true, "warm": "hms", "other": "tantivy", "factor": 0.010869565217391304, "factor_infinite": false, "ci": null, "verdict": "n/v", "text": "0.01x" } ]