"""Committed tests for iterative_evidence_convergence (iterate.md).
Engine tests run fully offline with fake search/extract functions.
Integration tests use a scripted stub client (rsearch-shaped search
plus chat), so the BlackCatBot tool, its alias, and the prompt-loop
path are covered without network.
"""
# retoor <retoor@molodetz.nl>
import json
import tempfile
import unittest
from blackcat.agent import BlackCatBot
from blackcat.converge import (
Claim,
EvidenceItem,
EvidenceState,
generate_initial_queries,
generate_refinement_queries,
iterative_evidence_convergence,
source_reliability,
update_confidence_and_graph,
)
from blackcat.fstate import FileState
from blackcat.tools import FileTools
TOPIC = "drinking coffee improves focus"
def make_search(counter):
domains = ["nature.com", "data.who.int", "reuters.com",
"example.org", "pubmed.ncbi.nlm.nih.gov",
"blog.example.com"]
def search(query, count):
counter["calls"] = counter.get("calls", 0) + 1
n = counter["calls"]
return {"results": [
{"title": "Study %d" % n,
"url": "https://%s/paper%d" % (domains[n % len(domains)], n),
"date": "2026-01-0%d" % (n % 9 + 1),
"summary": "%s: controlled evidence supports the claim"
% TOPIC},
{"title": "Data %d" % n,
"url": "https://site%d.example.org/d%d" % (n, n),
"date": "2026-02-01",
"summary": "%s: primary data supports the claim" % TOPIC}]}
return search
def extract(topic, state, fresh):
return {"claims": [{
"text": "Coffee improves focus",
"polarity": "supporting",
"supporting_evidence_ids": [e.id for e in fresh]}],
"contradictions": [], "open_questions": [],
"best_conclusion": "Coffee improves focus"}
class TestValidation(unittest.IsolatedAsyncioTestCase):
async def run_it(self, **kw):
kw.setdefault("search_fn", lambda q, c: {"results": []})
return await iterative_evidence_convergence(**kw)
async def test_bad_topic(self):
self.assertEqual((await self.run_it(topic=""))["code"], "bad_arguments")
self.assertEqual((await self.run_it(topic=None))["code"], "bad_arguments")
async def test_bad_iterations(self):
self.assertFalse((await self.run_it(topic="x", min_iterations=0))["ok"])
self.assertFalse((await self.run_it(
topic="x", min_iterations=5, max_iterations=3))["ok"])
self.assertFalse((await self.run_it(
topic="x", min_iterations=1, max_iterations=13))["ok"])
self.assertFalse((await self.run_it(
topic="x", min_iterations="3"))["ok"])
async def test_bad_threshold_and_no_backend(self):
self.assertFalse((await self.run_it(
topic="x", confidence_threshold=1.5))["ok"])
out = await iterative_evidence_convergence(topic="x", search_fn=None)
self.assertEqual(out["code"], "unavailable")
class TestScoring(unittest.IsolatedAsyncioTestCase):
async def test_reliability_tiers(self):
self.assertEqual(source_reliability("https://cdc.gov/x"), 0.92)
self.assertEqual(source_reliability("https://nature.com/a"), 0.80)
self.assertEqual(source_reliability("https://medium.com/p"), 0.30)
self.assertEqual(source_reliability("https://randomsite.io/a"),
0.55)
async def test_confidence_formula(self):
state = EvidenceState(topic="t")
claim = Claim(id="C1", text="t holds")
state.claims.append(claim)
items = [EvidenceItem("E1", "https://a.org", "s", 0.8, 1.0,
"2026-01-01", 0),
EvidenceItem("E2", "https://b.org", "s", 0.4, 1.0,
"2026-01-01", 0)]
state.evidence_items.extend(items)
state.claim_evidence_graph["C1"] = ["E1", "E2"]
state.evidence_stances = {"E1@C1": "supporting",
"E2@C1": "opposing"}
update_confidence_and_graph(state)
self.assertAlmostEqual(state.confidence_scores["C1"],
0.5909 + 0.075 + 0.05, places=3)
self.assertEqual((claim.support_count, claim.oppose_count), (1, 1))
async def test_initial_queries_four_angles_with_counter(self):
queries = generate_initial_queries(TOPIC)
self.assertEqual(len(queries), 4)
self.assertTrue(any("counterargument" in q for q in queries))
self.assertTrue(all(len(q) <= 128 for q in queries))
async def test_refinement_counter_quota_and_no_repeats(self):
state = EvidenceState(topic=TOPIC)
state.open_questions = ["does dose matter?"]
state.past_queries = generate_initial_queries(TOPIC)
queries = generate_refinement_queries(state, 1)
self.assertTrue(queries)
counter = [q for q in queries
if any(m in q.lower() for m in
("against", "criticism", "counterargument"))]
self.assertGreaterEqual(len(counter) / len(queries), 0.30)
self.assertFalse(set(queries) & set(state.past_queries))
class TestResolvedProblems(unittest.IsolatedAsyncioTestCase):
"""Regression tests for the iterative_converge.md analysis findings."""
def state_with_claim(self):
from blackcat.converge import Claim, EvidenceState
state = EvidenceState(topic=TOPIC)
claim = Claim(id="C1", text="Coffee improves focus")
state.claims.append(claim)
return state, claim
async def test_zero_evidence_claim_scores_zero(self):
from blackcat.converge import update_confidence_and_graph
state, claim = self.state_with_claim()
state.evidence_items.append(EvidenceItem(
"E1", "https://nature.com/a", TOPIC + " supports", 0.8, 1.0,
"2026-01-01", 0))
update_confidence_and_graph(state)
self.assertEqual(state.confidence_scores["C1"], 0.0)
async def test_contradiction_penalty_capped_and_unnamed_half(self):
from blackcat.converge import (
Contradiction,
update_confidence_and_graph,
)
state, claim = self.state_with_claim()
for i in range(2):
item = EvidenceItem("E%d" % i, "https://d%d.org/x" % i,
TOPIC + " supports", 0.9, 1.0,
"2026-01-01", 0)
state.evidence_items.append(item)
state.claim_evidence_graph.setdefault("C1", []).append(item.id)
state.evidence_stances[item.id + "@C1"] = "supporting"
for i in range(10):
state.contradictions.append(Contradiction(
id="X%d" % i, description="unnamed %d" % i, strength=0.9))
update_confidence_and_graph(state)
self.assertAlmostEqual(state.confidence_scores["C1"], 0.716,
places=2)
async def test_entropy_diversity_not_tanked_by_same_domain(self):
from blackcat.converge import compute_source_diversity
items = [EvidenceItem("E%d" % i, "https://d%d.org/x" % i, "s",
0.5, 0.5, None, 0) for i in range(4)]
self.assertAlmostEqual(compute_source_diversity(items), 0.75)
items += [EvidenceItem("X%d" % i, "https://d%d.org/y" % i, "s",
0.5, 0.5, None, 0) for i in range(4)]
self.assertAlmostEqual(compute_source_diversity(items), 0.75)
async def test_bm25_relevance_prefers_distinctive_match(self):
from blackcat.converge import BM25Index
items = [
EvidenceItem("E1", "https://a.org", "coffee coffee coffee "
"focus focus routine daily drink", 0.5, 0.1,
None, 0, title="daily"),
EvidenceItem("E2", "https://b.org", "caffeine adenosine "
"receptor antagonism vigilance study", 0.5, 0.1,
None, 0, title="mechanism")]
index = BM25Index(items)
self.assertTrue(index.available)
scores = index.scores("caffeine adenosine vigilance")
self.assertEqual(scores.get("E2"), 1.0)
async def test_claim_rephrase_merges_not_splits(self):
from blackcat.converge import EvidenceState, merge_extraction
state = EvidenceState(topic=TOPIC)
item1 = EvidenceItem("E1", "https://a.org", TOPIC, 0.8, 1.0,
None, 0)
item2 = EvidenceItem("E2", "https://b.org", TOPIC, 0.8, 1.0,
None, 1)
state.evidence_items.extend([item1, item2])
merge_extraction(state, {"claims": [{
"text": "Coffee improves focus",
"supporting_evidence_ids": ["E1"]}]}, [item1], 0)
merge_extraction(state, {"claims": [{
"text": "Coffee improves your focus and concentration",
"supporting_evidence_ids": ["E2"]}]}, [item2], 1)
self.assertEqual(len(state.claims), 1)
self.assertEqual(state.claim_evidence_graph["C1"], ["E1", "E2"])
async def test_ungrounded_link_dropped(self):
from blackcat.converge import EvidenceState, merge_extraction
state = EvidenceState(topic=TOPIC)
item = EvidenceItem("E1", "https://a.org",
"quantum bananas orbit Jupiter silently",
0.8, 1.0, None, 0)
state.evidence_items.append(item)
merge_extraction(state, {"claims": [{
"text": "Coffee improves focus",
"supporting_evidence_ids": ["E1"]}]}, [item], 0)
self.assertEqual(state.claim_evidence_graph.get("C1", []), [])
async def test_question_aging_to_insufficient_data(self):
from blackcat.converge import EvidenceState, age_open_questions
state = EvidenceState(topic=TOPIC)
state.open_questions = ["does dose matter for focus?"]
age_open_questions(state, ["does dose matter for focus effects"])
self.assertFalse(state.open_questions[0].startswith(
"insufficient data"))
age_open_questions(state, ["does dose matter for focus effects"])
self.assertTrue(state.open_questions[0].startswith(
"insufficient data:"))
async def test_dry_rounds_stop_early(self):
def same(query, count):
return {"results": [{
"title": "One", "url": "https://nature.com/only",
"date": "2026-01-01", "summary": TOPIC + " supports"}]}
out = await iterative_evidence_convergence(
TOPIC, min_iterations=1, max_iterations=10,
search_fn=same, extract_fn=extract)
self.assertEqual(out["stop_reason"], "dry_rounds")
self.assertLess(out["iterations"], 10)
async def test_counter_view_includes_minority_claims(self):
from blackcat.converge import (
Claim,
EvidenceState,
synthesize_conclusion,
update_confidence_and_graph,
)
state = EvidenceState(topic=TOPIC)
top = Claim(id="C1", text="Coffee improves focus")
minority = Claim(id="C2", text="Coffee harms sleep focus")
state.claims.extend([top, minority])
good = EvidenceItem("E1", "https://a.org", TOPIC, 0.9, 1.0,
"2026-01-01", 0)
bad = EvidenceItem("E2", "https://b.org", TOPIC + " harms", 0.85,
1.0, "2026-01-01", 0)
state.evidence_items.extend([good, bad])
state.claim_evidence_graph = {"C1": ["E1"], "C2": ["E2"]}
state.evidence_stances = {"E1@C1": "supporting",
"E2@C2": "opposing"}
update_confidence_and_graph(state)
out = synthesize_conclusion(state)
self.assertEqual([r["evidence_id"] for r in
out["counter_evidence"]], ["E2"])
self.assertEqual(out["confidence_kind"],
"heuristic_not_calibrated_probability")
self.assertTrue(out["conclusion"].startswith(
"Coffee improves focus"))
async def test_conclusion_picks_topic_claim_not_tangential(self):
from blackcat.converge import (
Claim,
EvidenceState,
synthesize_conclusion,
update_confidence_and_graph,
)
state = EvidenceState(topic=TOPIC)
main = Claim(id="C1", text="Coffee improves focus")
side = Claim(id="C2", text="Coffee is a brown drink")
state.claims.extend([main, side])
items = []
for i in range(4):
item = EvidenceItem("E%d" % i, "https://d%d.org/x" % i,
TOPIC + " study supports", 0.8, 1.0,
"2026-01-01", 0)
items.append(item)
state.claim_evidence_graph.setdefault("C1", []).append(
item.id)
state.evidence_stances[item.id + "@C1"] = "supporting"
side_item = EvidenceItem("E9", "https://z.org/x", "brown drink",
0.95, 1.0, "2026-01-01", 0)
items.append(side_item)
state.evidence_items.extend(items)
state.claim_evidence_graph["C2"] = ["E9"]
state.evidence_stances["E9@C2"] = "supporting"
update_confidence_and_graph(state)
out = synthesize_conclusion(state)
self.assertTrue(out["conclusion"].startswith(
"Coffee improves focus"))
self.assertIn("across all claims", out["conclusion"])
self.assertLess(state.confidence_scores["C2"], 0.85)
async def test_answered_questions_close_and_stop(self):
calls = {"n": 0}
def scripted(topic, state, fresh):
calls["n"] += 1
payload = extract(topic, state, fresh)
if calls["n"] == 1:
payload["open_questions"] = ["does dose matter?"]
else:
payload["answered_questions"] = ["does dose matter?"]
payload["open_questions"] = []
return payload
out = await iterative_evidence_convergence(
TOPIC, min_iterations=3, max_iterations=10,
search_fn=make_search({}), extract_fn=scripted)
self.assertTrue(out["stopped_early"])
self.assertEqual(out["stop_reason"], "converged")
self.assertEqual(out["answered_questions"], ["does dose matter?"])
self.assertEqual(out["unknown"], [])
class TestGenericImprovements(unittest.IsolatedAsyncioTestCase):
"""Regression tests for the israel2.md advice (topic-agnostic)."""
async def test_reliability_source_shape_caps(self):
from blackcat.converge import reliability_tier, source_reliability
self.assertEqual(source_reliability(
"https://studentnews.harvard.edu/article"), 0.60)
self.assertEqual(reliability_tier(
"https://studentnews.harvard.edu/article"),
"student_publication")
self.assertEqual(source_reliability(
"https://nytimes.com/opinion/essay"), 0.45)
self.assertEqual(reliability_tier(
"https://nytimes.com/opinion/essay"), "opinion")
self.assertEqual(source_reliability(
"https://repo.example.org/masterthesis.pdf"), 0.75)
self.assertEqual(reliability_tier(
"https://repo.example.org/masterthesis.pdf"), "thesis")
self.assertEqual(source_reliability("https://cdc.gov/x"), 0.92)
self.assertEqual(source_reliability("https://nature.com/a"), 0.80)
async def test_counter_status_and_stop_gate(self):
from blackcat.converge import EvidenceState, counter_evidence_status
state = EvidenceState(topic=TOPIC)
state.evidence_stances = {"E1@C1": "supporting"}
self.assertEqual(counter_evidence_status(state), "not_searched")
state.counter_rounds_attempted = 1
self.assertEqual(counter_evidence_status(state),
"scarce_despite_counter_search")
state.counter_rounds_attempted = 2
self.assertEqual(counter_evidence_status(state),
"exhausted_no_counter_found")
state.evidence_stances["E2@C1"] = "opposing"
state.evidence_stances.update(
{"E%d@C1" % i: "supporting" for i in range(3, 10)})
self.assertEqual(counter_evidence_status(state), "found")
async def test_main_question_unresolved_when_claim_is_tangential(self):
from blackcat.converge import (
Claim,
EvidenceState,
synthesize_conclusion,
update_confidence_and_graph,
)
state = EvidenceState(
topic="who started the river valley conflict, group A or "
"group B")
side = Claim(id="C1", text="the river valley has a long "
"history of trade")
state.claims.append(side)
item = EvidenceItem("E1", "https://a.org", "river valley trade "
"history group", 0.8, 1.0, "2026-01-01", 0)
state.evidence_items.append(item)
state.claim_evidence_graph = {"C1": ["E1"]}
state.evidence_stances = {"E1@C1": "supporting"}
update_confidence_and_graph(state)
out = synthesize_conclusion(state, stop_reason="max_iterations")
self.assertFalse(out["main_question_resolved"])
self.assertTrue(out["conclusion"].startswith(
"The main question remains unresolved"))
self.assertEqual(out["verdict_status"],
"incomplete_max_iterations")
self.assertEqual(len(out["single_source_claims"]), 1)
self.assertIn("single source", out["conclusion"])
async def test_one_sided_run_reports_counter_status(self):
out = await iterative_evidence_convergence(
TOPIC, min_iterations=1, max_iterations=2, early_stop=False,
search_fn=make_search({}), extract_fn=extract)
self.assertIn(out["counter_evidence_status"],
("scarce_despite_counter_search",
"exhausted_no_counter_found", "found"))
self.assertIn(out["verdict_status"],
("incomplete_max_iterations", "converged",
"incomplete_dry_rounds"))
class TestSectionD(unittest.IsolatedAsyncioTestCase):
"""Spec items (a)-(h) from the section-D research synthesis."""
async def test_a_anchor_frozen_and_drift_flagged(self):
calls = {"n": 0}
def shifting(topic, state, fresh):
calls["n"] += 1
payload = extract(topic, state, fresh)
payload["best_conclusion"] = \
"entirely different framing %d" % calls["n"]
return payload
out = await iterative_evidence_convergence(
TOPIC, min_iterations=1, max_iterations=3, early_stop=False,
search_fn=make_search({}), extract_fn=shifting)
self.assertTrue(out["anchor_text"])
state = out["state"]
self.assertEqual(state["anchor_text"], out["anchor_text"])
self.assertGreaterEqual(state["anchor_iteration"], 0)
async def test_b_triple_shows_dilution_overall_kept(self):
out = await iterative_evidence_convergence(
TOPIC, min_iterations=1, max_iterations=2, early_stop=False,
search_fn=make_search({}), extract_fn=extract)
self.assertIn("confidence_headline", out)
self.assertIn("confidence_coverage", out)
self.assertEqual(out["confidence"], out["overall_confidence"])
async def test_c_exhausted_after_consecutive_zero_yield(self):
from blackcat.converge import EvidenceState, counter_evidence_exhausted
state = EvidenceState(topic=TOPIC)
state.counter_rounds_attempted = 3
state.counter_rounds_with_opposing = 1
state.evidence_stances = {"E1@C1": "supporting"}
self.assertFalse(counter_evidence_exhausted(state))
state.counter_consecutive_zero = 2
self.assertTrue(counter_evidence_exhausted(state))
async def test_d_distinct_domains_and_sole_primary(self):
from blackcat.converge import Claim, EvidenceState, distinct_domains
state = EvidenceState(topic=TOPIC)
claim = Claim(id="C1", text=TOPIC)
state.claims.append(claim)
for i, url in enumerate(("https://a.org/1", "https://a.org/2",
"https://b.org/1")):
item = EvidenceItem("E%d" % i, url, TOPIC, 0.8, 1.0, None, 0)
state.evidence_items.append(item)
state.claim_evidence_graph.setdefault("C1", []).append(
item.id)
self.assertEqual(distinct_domains(state, claim), 2)
async def test_e_tier_gap_and_primary_query(self):
from blackcat.converge import Claim, EvidenceState, tier_gap
state = EvidenceState(topic=TOPIC)
claim = Claim(id="C1", text=TOPIC)
state.claims.append(claim)
item = EvidenceItem("E1", "https://randomblog.io/post", TOPIC,
0.3, 1.0, None, 0)
state.evidence_items.append(item)
state.claim_evidence_graph = {"C1": ["E1"]}
state.evidence_stances = {"E1@C1": "supporting"}
state.confidence_scores = {"C1": 0.9}
self.assertTrue(tier_gap(state)["gap"])
async def test_f_facet_ledger_in_result(self):
out = await iterative_evidence_convergence(
TOPIC, min_iterations=1, max_iterations=1,
search_fn=make_search({}), extract_fn=extract)
self.assertGreaterEqual(out["facet_coverage"]["total"], 1)
self.assertIn(out["facet_coverage"]["facets"][TOPIC],
("supported", "conflicted", "searched_none",
"uncovered"))
async def test_g_seeds(self):
counter = {}
out = await iterative_evidence_convergence(
TOPIC, min_iterations=1, max_iterations=1,
search_fn=make_search(counter), extract_fn=extract,
seed_queries=["known author study on coffee focus"],
seed_sources=[{"url": "https://nature.com/seeded",
"title": "Seeded",
"snippet": TOPIC + " seeded primary"}])
self.assertEqual(out["seeds_used"],
{"queries": 1, "sources": 1})
self.assertTrue(any("seeded" in e["source"] for e in
out["state"]["evidence_items"]))
bad = await iterative_evidence_convergence(
TOPIC, search_fn=make_search({}), seed_queries="nope")
self.assertEqual(bad["code"], "bad_arguments")
async def test_h_unicode_and_english_substitution(self):
from blackcat.converge import (
_tokens,
generate_initial_queries,
is_non_english_topic,
)
self.assertTrue(_tokens("Качество сна и кофе"))
self.assertTrue(is_non_english_topic(
"wie is de schuldige van het conflict"))
queries = generate_initial_queries(
"wie is de schuldige van het conflict",
topic_en="who is responsible for the conflict")
self.assertEqual(len(queries), 4)
self.assertTrue(any("who is responsible" in q for q in queries))
english = generate_initial_queries(TOPIC, topic_en="ignored")
self.assertTrue(any("experts" in q for q in english))
GATE_CLAIM = (
"The origins of the Israeli-Palestinian conflict lie in the struggle "
"over land and labor markets on the frontier of Zionist settlement in "
"late Ottoman Palestine, especially during the first and second "
"aliyot (1882-1914).")
TOPIC_NL = ("Wie is de oorspronkelijke veroorzaker / starter van het "
"Israëlisch-Palestijnse (Arabisch-Israëlische) conflict? "
"Historische oorsprong rond 1880-1948.")
TOPIC_EN = ("Who originally started the Israeli-Palestinian conflict? "
"Historical origins around 1880-1948.")
class TestProposals(unittest.IsolatedAsyncioTestCase):
"""P1-P10 from converge_proposals.md (Gate 0 numbers: NL 0.0625,
EN 0.40 against GATE_CLAIM, before the fix)."""
async def test_gate0_relevance_with_normalised_scoring(self):
from blackcat.converge import (
Claim,
EvidenceState,
claim_relevance_tier,
)
raw = EvidenceState(topic=TOPIC_NL)
claim = Claim(id="C1", text=GATE_CLAIM)
self.assertEqual(claim_relevance_tier(raw, claim), 0.0)
prepared = EvidenceState(topic=TOPIC_NL)
prepared.scoring_topic = TOPIC_EN
prepared.topic_keywords = ["Israeli-Palestinian conflict",
"land", "labor markets",
"Zionist settlement", "Ottoman"]
self.assertGreaterEqual(claim_relevance_tier(prepared, claim),
0.5)
unrelated = Claim(id="C2", text="Iron is the most common "
"element in Earth's core")
self.assertEqual(claim_relevance_tier(prepared, unrelated), 0.0)
async def test_p2_counter_classification_strips_topic(self):
from blackcat.converge import (
EvidenceState,
_is_counter,
generate_refinement_queries,
)
self.assertFalse(_is_counter(
"what is the evidence for why deterrence fails",
"why deterrence fails"))
self.assertTrue(_is_counter(
"counterarguments against why deterrence fails",
"why deterrence fails"))
state = EvidenceState(topic="why deterrence fails")
queries = generate_refinement_queries(state, 1)
counter = [q for q in queries
if _is_counter(q, state.topic)]
self.assertGreaterEqual(len(counter) / len(queries), 0.30)
async def test_p3_comparison_mode_divided_verdict(self):
from blackcat.converge import (
Claim,
EvidenceState,
synthesize_conclusion,
update_confidence_and_graph,
)
state = EvidenceState(topic="who started the valley conflict")
state.candidates = [
{"name": "Group A first move", "description": "A acted"},
{"name": "Group B first move", "description": "B acted"}]
for i, (cid, text) in enumerate((
("C1", "Group A first move started the valley conflict"),
("C2", "Group B first move started the valley conflict"))):
claim = Claim(id=cid, text=text)
state.claims.append(claim)
item = EvidenceItem("E%d" % i, "https://d%d.org/x" % i,
text, 0.8, 1.0, "2026-01-01", 0)
state.evidence_items.append(item)
state.claim_evidence_graph[cid] = [item.id]
state.evidence_stances[item.id + "@" + cid] = "supporting"
update_confidence_and_graph(state)
out = synthesize_conclusion(state)
self.assertTrue(out["comparison_mode"])
self.assertEqual(out["verdict_status"],
"divided_no_dominant_candidate")
self.assertEqual(len(out["candidates"]), 2)
self.assertIn("literature divides", out["conclusion"])
async def test_p4_anchor_gate(self):
from blackcat.converge import (
Claim,
EvidenceState,
synthesize_conclusion,
update_confidence_and_graph,
)
state = EvidenceState(topic="who started the valley conflict")
state.anchor_claim_id = "C1"
state.anchor_text = "unrelated baking recipes of the valley"
claim = Claim(id="C1", text="unrelated baking recipes of the "
"valley")
state.claims.append(claim)
item = EvidenceItem("E1", "https://a.org", "valley baking", 0.8,
1.0, "2026-01-01", 0)
state.evidence_items.append(item)
state.claim_evidence_graph = {"C1": ["E1"]}
state.evidence_stances = {"E1@C1": "supporting"}
state.current_best_conclusion = \
"who started the valley conflict group A"
update_confidence_and_graph(state)
out = synthesize_conclusion(state)
self.assertEqual(out["verdict_status"],
"answered_adjacent_question")
self.assertEqual(out["adjacent_question"], state.anchor_text)
self.assertIn("adjacent question", out["conclusion"])
async def test_p5_full_stance_set_and_sentence_boundary(self):
out = await iterative_evidence_convergence(
TOPIC, min_iterations=1, max_iterations=2, early_stop=False,
search_fn=make_search({}), extract_fn=extract)
self.assertIn("evidence_by_stance", out)
self.assertTrue(out["conclusion"].rstrip().endswith(
(".", ")", '"')) or out["conclusion"].endswith("."))
async def test_p6_ineffective_counter_search_not_exhausted(self):
from blackcat.converge import EvidenceState, counter_evidence_status
state = EvidenceState(topic=TOPIC)
state.counter_rounds_attempted = 3
state.counter_query_relevances = [0.01, 0.02]
state.evidence_stances = {"E1@C1": "supporting"}
self.assertEqual(counter_evidence_status(state),
"counter_search_ineffective")
async def test_p7_facet_cap_and_topic_slot(self):
out = await iterative_evidence_convergence(
TOPIC, min_iterations=1, max_iterations=2, early_stop=False,
search_fn=make_search({}), extract_fn=extract)
self.assertLessEqual(out["facet_coverage"]["total"], 8)
queries = [q for entry in out["iteration_history"]
for q in entry["queries"]]
self.assertIn(TOPIC, queries)
async def test_p9_seeds_dropped_reported(self):
out = await iterative_evidence_convergence(
TOPIC, min_iterations=1, max_iterations=1,
search_fn=make_search({}), extract_fn=extract,
seed_queries=["s1 coffee", "s2 coffee", "s3 coffee",
"s4 coffee", "s5 coffee", "s5 coffee"])
self.assertEqual(out["seeds_used"]["queries"], 4)
reasons = {d["reason"] for d in out["seeds_dropped"]}
self.assertIn("duplicate", reasons)
self.assertTrue(any(r.startswith("round_cap") for r in reasons))
self.assertIn("run_fingerprint", out)
self.assertTrue(out["run_fingerprint"].startswith("sha256:"))
async def test_p10_cache_reuses_search(self):
counter = {}
cache = {}
search = make_search(counter)
await iterative_evidence_convergence(
TOPIC, min_iterations=1, max_iterations=1, search_fn=search,
extract_fn=extract, search_cache=cache)
first = counter["calls"]
await iterative_evidence_convergence(
TOPIC, min_iterations=1, max_iterations=1, search_fn=search,
extract_fn=extract, search_cache=cache)
self.assertEqual(counter["calls"], first)
class TestLoop(unittest.IsolatedAsyncioTestCase):
async def test_early_stop_after_min_iterations(self):
counter = {}
out = await iterative_evidence_convergence(
TOPIC, min_iterations=3, max_iterations=10,
search_fn=make_search(counter), extract_fn=extract)
self.assertTrue(out["ok"])
self.assertEqual(out["topic"], TOPIC)
self.assertEqual(out["iterations"], 3)
self.assertTrue(out["stopped_early"])
self.assertGreaterEqual(out["confidence"], 0.80)
self.assertIn("Coffee improves focus", out["conclusion"])
self.assertTrue(out["supporting_evidence"])
self.assertEqual(out["unknown"], [])
self.assertEqual(len(out["iteration_history"]), 3)
state = out["state"]
for field_name in ("claims", "evidence_items",
"claim_evidence_graph", "confidence_scores",
"contradictions", "open_questions",
"source_diversity", "iteration_log",
"current_best_conclusion", "overall_confidence"):
self.assertIn(field_name, state)
self.assertGreaterEqual(state["source_diversity"], 0.65)
async def test_max_iterations_hard_cap_without_early_stop(self):
out = await iterative_evidence_convergence(
TOPIC, min_iterations=1, max_iterations=4, early_stop=False,
search_fn=make_search({}), extract_fn=extract)
self.assertEqual(out["iterations"], 4)
self.assertFalse(out["stopped_early"])
async def test_fallback_extractor_without_llm(self):
out = await iterative_evidence_convergence(
TOPIC, min_iterations=1, max_iterations=2,
search_fn=make_search({}))
self.assertTrue(out["ok"])
self.assertTrue(out["claims"])
self.assertGreater(out["evidence_count"], 0)
async def test_search_failure_degrades_not_crashes(self):
def boom(query, count):
raise RuntimeError("search down")
out = await iterative_evidence_convergence(
TOPIC, min_iterations=1, max_iterations=2, search_fn=boom)
self.assertTrue(out["ok"])
self.assertEqual(out["evidence_count"], 0)
self.assertIn("Insufficient evidence", out["conclusion"])
async def test_dedup_across_rounds(self):
def same(query, count):
return {"results": [{
"title": "One", "url": "https://nature.com/only",
"date": "2026-01-01",
"summary": "%s supports" % TOPIC}]}
out = await iterative_evidence_convergence(
TOPIC, min_iterations=1, max_iterations=3, early_stop=False,
search_fn=same, extract_fn=extract)
self.assertEqual(out["evidence_count"], 1)
def _resp(message):
return {"choices": [{"message": message, "finish_reason": "stop"}],
"usage": {"prompt_tokens": 5}}
class StubDeepSeek:
"""Scripted client: rsearch-shaped search + extractor chat.
The ``search`` attribute doubles as the bot's ``_rsearch``
backend (the only search backend); ``chat`` serves the prompt
loop, the prepare step, and extraction.
"""
model = "stub-model"
def __init__(self):
self.counter = {}
self._sync_search = make_search(self.counter)
self.main_calls = 0
async def search(self, query, count=10):
return self._sync_search(query, count)
async def chat(self, messages, **kwargs):
system = messages[0].get("content", "") if messages else ""
if "Prepare a research question" in system:
return _resp({"role": "assistant", "content": json.dumps({
"language": "en",
"normalized_question": TOPIC,
"keywords": ["coffee", "focus", "caffeine"],
"question_type": "factual", "candidates": []})})
if "extract structured evidence" in system:
payload = json.loads(messages[1]["content"])
ids = [e["id"] for e in payload["new_evidence"]]
return _resp({"role": "assistant", "content": json.dumps({
"claims": [{"text": "Coffee improves focus",
"polarity": "supporting",
"supporting_evidence_ids": ids}],
"contradictions": [], "open_questions": [],
"best_conclusion": "Coffee improves focus"})})
if kwargs.get("tools"):
self.main_calls += 1
if self.main_calls == 1:
return {"choices": [{"message": {
"role": "assistant", "content": None,
"tool_calls": [{
"id": "c1", "type": "function",
"function": {
"name": "iterative_evidence_convergence",
"arguments": json.dumps({
"topic": TOPIC, "min_iterations": 1,
"max_iterations": 2,
"early_stop": False})}}]},
"finish_reason": "tool_calls"}],
"usage": {"prompt_tokens": 5}}
return _resp({"role": "assistant",
"content": "Converged: coffee improves focus"})
return _resp({"role": "assistant", "content": "unused"})
class TestBotIntegration(unittest.IsolatedAsyncioTestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.addCleanup(self.tmp.cleanup)
def bot(self):
bot = BlackCatBot(provider="deepseek", api_key="k", system="t",
tools=FileTools(FileState(self.tmp.name)))
bot._client = StubDeepSeek()
bot._rsearch = bot._client
self.addAsyncCleanup(bot.aclose)
return bot
async def test_run_tool_converges(self):
bot = self.bot()
out = await bot.run_tool("iterative_evidence_convergence",
topic=TOPIC, min_iterations=1,
max_iterations=2, early_stop=False)
self.assertTrue(out["ok"])
self.assertEqual(out["iterations"], 2)
self.assertIn("Coffee improves focus", out["conclusion"])
self.assertTrue(out["supporting_evidence"])
self.assertIn("evidence_count", out)
self.assertNotIn("state", out)
self.assertGreater(out["state_summary"]["evidence_items"], 0)
self.assertTrue(out["english_scoring_used"])
async def test_alias_converge_on_claim(self):
bot = self.bot()
out = await bot.run_tool("converge_on_claim", topic=TOPIC,
min_iterations=1, max_iterations=1)
self.assertTrue(out["ok"])
async def test_bad_arguments_returned_not_raised(self):
bot = self.bot()
out = await bot.run_tool("iterative_evidence_convergence", topic="",
min_iterations=1, max_iterations=1)
self.assertEqual(out["code"], "bad_arguments")
async def test_echo_provider_unavailable(self):
async with BlackCatBot(provider="echo", system="t") as bot:
out = await bot.run_tool("iterative_evidence_convergence",
topic=TOPIC)
self.assertEqual(out["code"], "unavailable")
async def test_openrouter_provider_converges(self):
bot = BlackCatBot(provider="openrouter", api_key="k", system="t",
tools=FileTools(FileState(self.tmp.name)))
bot._client = StubDeepSeek()
bot._rsearch = bot._client
self.addAsyncCleanup(bot.aclose)
out = await bot.run_tool("iterative_evidence_convergence",
topic=TOPIC, min_iterations=1,
max_iterations=1)
self.assertTrue(out["ok"])
async def test_prompt_loop_calls_converge_tool(self):
bot = self.bot()
reply = await bot.prompt("Is it true that %s?" % TOPIC)
self.assertEqual(reply, "Converged: coffee improves focus")
tool_msgs = [m for m in bot.messages if m.get("role") == "tool"]
self.assertEqual(len(tool_msgs), 1)
frames = [f for f in bot.transcript if f["kind"] == "tool"]
self.assertTrue(frames[-1]["out"]["ok"])
self.assertEqual(frames[-1]["tool"],
"iterative_evidence_convergence")
self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages()))
if __name__ == "__main__":
unittest.main()