# Claims about kmerseek, each with the test that would kill it.
#
# Nothing here is a typed-in result. A `table` test is recomputed from
# data/results.parquet on every build. A `record` test quotes one result of the
# experiment inventory (data/experiments.json) and the build fails if the quote
# no longer matches that record. Status comes from the test, not from this file:
# a claim is alive when its test runs and does not kill it, retired when it does,
# untested when the test has not been run or the data cannot decide it.
# A test's optional `verdict` is the wording of its verdict line, a Python format
# string filled from the resolved test (record_compare gives a, b, a_plus_b, and, with
# `total`, total and a_plus_b_share).
# Schema: claims/schema.json. Check: scripts/claims.py.

tool: kmerseek
title: kmerseek
updated: "2026-10-15"

claims:
  - id: phmmer-parity-mammals
    claim: Parity with phmmer inside mammals
    kill: Fmax more than 0.05 from phmmer's on any of the 6 mammals
    scope: Swiss-Prot features, raw calls, setting hp_thomas_dill2_k24_lcFalse, 998 human queries
    test:
      kind: table
      experiment: qfo_region_midi_plus
      truth_source: Swiss-Prot features
      dedup_or_raw: raw
      stratum_axis: all
      stratum_bin: all
      metric: fmax
      tool: kmerseek
      setting: hp_thomas_dill2_k24_lcFalse
      against: {tool: hmmer3_phmmer, setting: default}
      targets: [QfO ptroglodytes, QfO rnorvegicus, QfO clupus, QfO btaurus, QfO mouse, QfO mdomestica]
      kill_if: {abs_difference_above: 0.05}
    context:
      - label: on Pfam domains, the best kmerseek setting for each mammal
        test:
          kind: table
          experiment: qfo_region_midi_plus
          truth_source: Pfam-A domain instances
          dedup_or_raw: raw
          stratum_axis: all
          stratum_bin: all
          metric: fmax
          tool: kmerseek
          setting: best_per_target
          against: {tool: hmmer3_phmmer, setting: default}
          targets: [QfO ptroglodytes, QfO rnorvegicus, QfO clupus, QfO btaurus, QfO mouse, QfO mdomestica]
          kill_if: {abs_difference_above: 0.05}
      - label: settings that pass on Swiss-Prot features (raw or dedup)
        test:
          kind: table_count_passing
          experiment: qfo_region_midi_plus
          truth_source: Swiss-Prot features
          stratum_axis: all
          stratum_bin: all
          metric: fmax
          tool: kmerseek
          against: {tool: hmmer3_phmmer, setting: default}
          targets: [QfO ptroglodytes, QfO rnorvegicus, QfO clupus, QfO btaurus, QfO mouse, QfO mdomestica]
          kill_if: {abs_difference_above: 0.05}
    evidence: {label: leaderboard, href: index.html}

  - id: hp-seeds-below-40
    claim: Exact HP seeds find homologs below 40% identity
    kill: Fmax under 0.05 in the under-40% identity bins on Pfam-N, for every hydrophobic-polar setting
    scope: Pfam-N regions, raw calls, kmerseek hp_thomas_dill2 settings, all 15 relatives
    test:
      kind: table_bins
      experiment: qfo_region_midi_plus
      truth_source: Pfam-N domain instances
      dedup_or_raw: raw
      stratum_axis: identity
      stratum_bins: ["0-20%", "20-30%", "30-40%"]
      metric: fmax
      tool: kmerseek
      setting_prefix: hp_
      min_instances: 30
      kill_if: {all_below: 0.05}
    context:
      - label: share of Pfam pairs at 20-30% identity sharing one exact hp_thomas_dill2 23-mer
        test: {kind: record, row: "230", name: "share of Pfam 20-30% pairs sharing an exact hp_thomas_dill2 23-mer (column 23)", quote: "0.038"}
      - label: recall at a 0.001 false-positive rate for an exact 23-mer seed, Pfam 20-30%
        test: {kind: record, row: "232", name: "recall at null FPR 0.001, exact 23-mer (seed 23), Pfam 20-30%", quote: "0.039"}
    evidence: {label: nb 230 to 232, href: "https://github.com/seanome/2024-kmerseek-analysis/blob/fe72334b6c6c34ae19728ae292ad0ff1c4ae95d9/data/experiments.json"}

  - id: faster-than-mmseqs2
    claim: Faster per query than MMseqs2
    kill: kmerseek answers fewer queries per second than MMseqs2 on the same 1,000 Swiss-Prot queries
    scope: 1,000 Swiss-Prot queries against Swiss-Prot on one laptop; kmerseek hp k15, scaled 10
    test:
      kind: record_compare
      row: "239"
      ours: {name: "kmerseek search queries per second", quote: "2.289115"}
      theirs: {name: "MMseqs2 queries per second", quote: "78.554595"}
      kill_if: {ours_below_theirs: true}
      verdict: "{a:.2f} queries per second against MMseqs2's {b:.1f}"
    context:
      - label: kmerseek search peak memory, GB
        test: {kind: record, row: "239", name: "kmerseek hp k15 scaled10 search peak memory (GB)", quote: "82.68"}
      - label: MMseqs2 -s 5.7 search peak memory, GB
        test: {kind: record, row: "239", name: "MMseqs2 -s 5.7 search peak memory (GB)", quote: "2.55"}
    evidence: {label: nb 239, href: "https://github.com/seanome/2024-kmerseek-analysis/blob/ba20e8b/notebooks/239_laptop_scale_swissprot.ipynb"}

  - id: beats-foldseek-scope
    claim: Outperforms Foldseek on SCOPe
    kill: inside FoldSeek's 5,713-query SCOPe40 benchmark, kmerseek detects no query that FoldSeek misses
    scope: SCOPe40, hp_thomas_dill at k 26, family level
    test:
      kind: record_compare
      row: "076"
      ours: {name: "benchmark queries only kmerseek detects", quote: "0"}
      theirs: {name: "benchmark queries both detect", quote: "188"}
      total: {field: dataset, quote: "5,713"}
      kill_if: {ours_at_most: 0}
      verdict: "fired on {a_plus_b:,.0f} of {total:,.0f} benchmark queries ({a_plus_b_share:.1%}); {a:,.0f} that FoldSeek missed"
    context:
      - label: per-query family sensitivity, kmerseek better than FoldSeek on 228 shared queries (Wilcoxon)
        test: {kind: record, row: "076", name: "Wilcoxon p, kmerseek better than FoldSeek", quote: "p=0.1171"}
      - label: best kmerseek 0.4 AUC over all 5,713 reference queries (family), any setting
        test: {kind: table_max, experiment: scope40_v040, metric: auc_ref, truth_source: SCOPe family, stratum_axis: all, tool: kmerseek}
      - label: best kmerseek 0.4 AUC over only the queries it fires on (family), any setting
        test: {kind: table_max, experiment: scope40_v040, metric: auc_all, truth_source: SCOPe family, stratum_axis: all, tool: kmerseek}
    evidence: {label: nb 076, href: "https://github.com/seanome/2024-kmerseek-analysis/blob/31f53c1/notebooks/076_hp_alphabet_sweep_per_class.ipynb"}

  - id: short-features-placed
    claim: Short features under 60 aa placed at 50 to 90% identity
    kill: a Kyte-Doolittle hydrophobicity window, with no search, lands on as many of the 87 cases
    scope: the 87 short Swiss-Prot features of nb 244 where kmerseek drew the boundary and every other tool's call ran past it
    test: {kind: not_run, reason: "the Kyte-Doolittle window was not run on the 87 cases"}
    context:
      - label: short features passing every criterion in nb 244
        test: {kind: record, row: "244", name: "cases_passing_every_criterion", quote: "87"}
      - label: across all Swiss-Prot features, not only short ones, kmerseek settings above a Kyte-Doolittle scan
        test: {kind: record, row: "231", name: "kmerseek arms above a Kyte-Doolittle scan (no search) on all features", quote: "0 of 406"}
    evidence: {label: nb 244, href: "https://github.com/seanome/2024-kmerseek-analysis/blob/fe72334b6c6c34ae19728ae292ad0ff1c4ae95d9/data/experiments.json"}

  - id: bhf-tether
    claim: The BHF membrane tether is a find other tools miss
    kill: a Kyte-Doolittle window on BHF calls the same region
    scope: Botryllus histocompatibility factor (BHF)
    test: {kind: not_run, reason: "no Kyte-Doolittle window has been run on BHF"}
    context:
      - label: best E-value of any human protein for BHF, any kmerseek setting (nb 241)
        test: {kind: record, row: "241", name: "bhf_best_evalue_any_human_protein", quote: "0.609548"}
    evidence: {label: nb 241, href: "https://github.com/seanome/2024-kmerseek-analysis/blob/fe72334b6c6c34ae19728ae292ad0ff1c4ae95d9/data/experiments.json"}