"""Committed tests for iterative_evidence_convergence (iterate.md). Engine tests run fully offline with fake search/extract functions. Integration tests use a scripted stub client (rsearch-shaped search plus chat), so the BlackCatBot tool, its alias, and the prompt-loop path are covered without network. """ # retoor import json import tempfile import unittest from blackcat.agent import BlackCatBot from blackcat.converge import ( Claim, EvidenceItem, EvidenceState, generate_initial_queries, generate_refinement_queries, iterative_evidence_convergence, source_reliability, update_confidence_and_graph, ) from blackcat.fstate import FileState from blackcat.tools import FileTools TOPIC = "drinking coffee improves focus" def make_search(counter): domains = ["nature.com", "data.who.int", "reuters.com", "example.org", "pubmed.ncbi.nlm.nih.gov", "blog.example.com"] def search(query, count): counter["calls"] = counter.get("calls", 0) + 1 n = counter["calls"] return {"results": [ {"title": "Study %d" % n, "url": "https://%s/paper%d" % (domains[n % len(domains)], n), "date": "2026-01-0%d" % (n % 9 + 1), "summary": "%s: controlled evidence supports the claim" % TOPIC}, {"title": "Data %d" % n, "url": "https://site%d.example.org/d%d" % (n, n), "date": "2026-02-01", "summary": "%s: primary data supports the claim" % TOPIC}]} return search def extract(topic, state, fresh): return {"claims": [{ "text": "Coffee improves focus", "polarity": "supporting", "supporting_evidence_ids": [e.id for e in fresh]}], "contradictions": [], "open_questions": [], "best_conclusion": "Coffee improves focus"} class TestValidation(unittest.IsolatedAsyncioTestCase): async def run_it(self, **kw): kw.setdefault("search_fn", lambda q, c: {"results": []}) return await iterative_evidence_convergence(**kw) async def test_bad_topic(self): self.assertEqual((await self.run_it(topic=""))["code"], "bad_arguments") self.assertEqual((await self.run_it(topic=None))["code"], "bad_arguments") async def test_bad_iterations(self): self.assertFalse((await self.run_it(topic="x", min_iterations=0))["ok"]) self.assertFalse((await self.run_it( topic="x", min_iterations=5, max_iterations=3))["ok"]) self.assertFalse((await self.run_it( topic="x", min_iterations=1, max_iterations=13))["ok"]) self.assertFalse((await self.run_it( topic="x", min_iterations="3"))["ok"]) async def test_bad_threshold_and_no_backend(self): self.assertFalse((await self.run_it( topic="x", confidence_threshold=1.5))["ok"]) out = await iterative_evidence_convergence(topic="x", search_fn=None) self.assertEqual(out["code"], "unavailable") class TestScoring(unittest.IsolatedAsyncioTestCase): async def test_reliability_tiers(self): self.assertEqual(source_reliability("https://cdc.gov/x"), 0.92) self.assertEqual(source_reliability("https://nature.com/a"), 0.80) self.assertEqual(source_reliability("https://medium.com/p"), 0.30) self.assertEqual(source_reliability("https://randomsite.io/a"), 0.55) async def test_confidence_formula(self): state = EvidenceState(topic="t") claim = Claim(id="C1", text="t holds") state.claims.append(claim) items = [EvidenceItem("E1", "https://a.org", "s", 0.8, 1.0, "2026-01-01", 0), EvidenceItem("E2", "https://b.org", "s", 0.4, 1.0, "2026-01-01", 0)] state.evidence_items.extend(items) state.claim_evidence_graph["C1"] = ["E1", "E2"] state.evidence_stances = {"E1@C1": "supporting", "E2@C1": "opposing"} update_confidence_and_graph(state) self.assertAlmostEqual(state.confidence_scores["C1"], 0.5909 + 0.075 + 0.05, places=3) self.assertEqual((claim.support_count, claim.oppose_count), (1, 1)) async def test_initial_queries_four_angles_with_counter(self): queries = generate_initial_queries(TOPIC) self.assertEqual(len(queries), 4) self.assertTrue(any("counterargument" in q for q in queries)) self.assertTrue(all(len(q) <= 128 for q in queries)) async def test_refinement_counter_quota_and_no_repeats(self): state = EvidenceState(topic=TOPIC) state.open_questions = ["does dose matter?"] state.past_queries = generate_initial_queries(TOPIC) queries = generate_refinement_queries(state, 1) self.assertTrue(queries) counter = [q for q in queries if any(m in q.lower() for m in ("against", "criticism", "counterargument"))] self.assertGreaterEqual(len(counter) / len(queries), 0.30) self.assertFalse(set(queries) & set(state.past_queries)) class TestResolvedProblems(unittest.IsolatedAsyncioTestCase): """Regression tests for the iterative_converge.md analysis findings.""" def state_with_claim(self): from blackcat.converge import Claim, EvidenceState state = EvidenceState(topic=TOPIC) claim = Claim(id="C1", text="Coffee improves focus") state.claims.append(claim) return state, claim async def test_zero_evidence_claim_scores_zero(self): from blackcat.converge import update_confidence_and_graph state, claim = self.state_with_claim() state.evidence_items.append(EvidenceItem( "E1", "https://nature.com/a", TOPIC + " supports", 0.8, 1.0, "2026-01-01", 0)) update_confidence_and_graph(state) self.assertEqual(state.confidence_scores["C1"], 0.0) async def test_contradiction_penalty_capped_and_unnamed_half(self): from blackcat.converge import ( Contradiction, update_confidence_and_graph, ) state, claim = self.state_with_claim() for i in range(2): item = EvidenceItem("E%d" % i, "https://d%d.org/x" % i, TOPIC + " supports", 0.9, 1.0, "2026-01-01", 0) state.evidence_items.append(item) state.claim_evidence_graph.setdefault("C1", []).append(item.id) state.evidence_stances[item.id + "@C1"] = "supporting" for i in range(10): state.contradictions.append(Contradiction( id="X%d" % i, description="unnamed %d" % i, strength=0.9)) update_confidence_and_graph(state) self.assertAlmostEqual(state.confidence_scores["C1"], 0.716, places=2) async def test_entropy_diversity_not_tanked_by_same_domain(self): from blackcat.converge import compute_source_diversity items = [EvidenceItem("E%d" % i, "https://d%d.org/x" % i, "s", 0.5, 0.5, None, 0) for i in range(4)] self.assertAlmostEqual(compute_source_diversity(items), 0.75) items += [EvidenceItem("X%d" % i, "https://d%d.org/y" % i, "s", 0.5, 0.5, None, 0) for i in range(4)] self.assertAlmostEqual(compute_source_diversity(items), 0.75) async def test_bm25_relevance_prefers_distinctive_match(self): from blackcat.converge import BM25Index items = [ EvidenceItem("E1", "https://a.org", "coffee coffee coffee " "focus focus routine daily drink", 0.5, 0.1, None, 0, title="daily"), EvidenceItem("E2", "https://b.org", "caffeine adenosine " "receptor antagonism vigilance study", 0.5, 0.1, None, 0, title="mechanism")] index = BM25Index(items) self.assertTrue(index.available) scores = index.scores("caffeine adenosine vigilance") self.assertEqual(scores.get("E2"), 1.0) async def test_claim_rephrase_merges_not_splits(self): from blackcat.converge import EvidenceState, merge_extraction state = EvidenceState(topic=TOPIC) item1 = EvidenceItem("E1", "https://a.org", TOPIC, 0.8, 1.0, None, 0) item2 = EvidenceItem("E2", "https://b.org", TOPIC, 0.8, 1.0, None, 1) state.evidence_items.extend([item1, item2]) merge_extraction(state, {"claims": [{ "text": "Coffee improves focus", "supporting_evidence_ids": ["E1"]}]}, [item1], 0) merge_extraction(state, {"claims": [{ "text": "Coffee improves your focus and concentration", "supporting_evidence_ids": ["E2"]}]}, [item2], 1) self.assertEqual(len(state.claims), 1) self.assertEqual(state.claim_evidence_graph["C1"], ["E1", "E2"]) async def test_ungrounded_link_dropped(self): from blackcat.converge import EvidenceState, merge_extraction state = EvidenceState(topic=TOPIC) item = EvidenceItem("E1", "https://a.org", "quantum bananas orbit Jupiter silently", 0.8, 1.0, None, 0) state.evidence_items.append(item) merge_extraction(state, {"claims": [{ "text": "Coffee improves focus", "supporting_evidence_ids": ["E1"]}]}, [item], 0) self.assertEqual(state.claim_evidence_graph.get("C1", []), []) async def test_question_aging_to_insufficient_data(self): from blackcat.converge import EvidenceState, age_open_questions state = EvidenceState(topic=TOPIC) state.open_questions = ["does dose matter for focus?"] age_open_questions(state, ["does dose matter for focus effects"]) self.assertFalse(state.open_questions[0].startswith( "insufficient data")) age_open_questions(state, ["does dose matter for focus effects"]) self.assertTrue(state.open_questions[0].startswith( "insufficient data:")) async def test_dry_rounds_stop_early(self): def same(query, count): return {"results": [{ "title": "One", "url": "https://nature.com/only", "date": "2026-01-01", "summary": TOPIC + " supports"}]} out = await iterative_evidence_convergence( TOPIC, min_iterations=1, max_iterations=10, search_fn=same, extract_fn=extract) self.assertEqual(out["stop_reason"], "dry_rounds") self.assertLess(out["iterations"], 10) async def test_counter_view_includes_minority_claims(self): from blackcat.converge import ( Claim, EvidenceState, synthesize_conclusion, update_confidence_and_graph, ) state = EvidenceState(topic=TOPIC) top = Claim(id="C1", text="Coffee improves focus") minority = Claim(id="C2", text="Coffee harms sleep focus") state.claims.extend([top, minority]) good = EvidenceItem("E1", "https://a.org", TOPIC, 0.9, 1.0, "2026-01-01", 0) bad = EvidenceItem("E2", "https://b.org", TOPIC + " harms", 0.85, 1.0, "2026-01-01", 0) state.evidence_items.extend([good, bad]) state.claim_evidence_graph = {"C1": ["E1"], "C2": ["E2"]} state.evidence_stances = {"E1@C1": "supporting", "E2@C2": "opposing"} update_confidence_and_graph(state) out = synthesize_conclusion(state) self.assertEqual([r["evidence_id"] for r in out["counter_evidence"]], ["E2"]) self.assertEqual(out["confidence_kind"], "heuristic_not_calibrated_probability") self.assertTrue(out["conclusion"].startswith( "Coffee improves focus")) async def test_conclusion_picks_topic_claim_not_tangential(self): from blackcat.converge import ( Claim, EvidenceState, synthesize_conclusion, update_confidence_and_graph, ) state = EvidenceState(topic=TOPIC) main = Claim(id="C1", text="Coffee improves focus") side = Claim(id="C2", text="Coffee is a brown drink") state.claims.extend([main, side]) items = [] for i in range(4): item = EvidenceItem("E%d" % i, "https://d%d.org/x" % i, TOPIC + " study supports", 0.8, 1.0, "2026-01-01", 0) items.append(item) state.claim_evidence_graph.setdefault("C1", []).append( item.id) state.evidence_stances[item.id + "@C1"] = "supporting" side_item = EvidenceItem("E9", "https://z.org/x", "brown drink", 0.95, 1.0, "2026-01-01", 0) items.append(side_item) state.evidence_items.extend(items) state.claim_evidence_graph["C2"] = ["E9"] state.evidence_stances["E9@C2"] = "supporting" update_confidence_and_graph(state) out = synthesize_conclusion(state) self.assertTrue(out["conclusion"].startswith( "Coffee improves focus")) self.assertIn("across all claims", out["conclusion"]) self.assertLess(state.confidence_scores["C2"], 0.85) async def test_answered_questions_close_and_stop(self): calls = {"n": 0} def scripted(topic, state, fresh): calls["n"] += 1 payload = extract(topic, state, fresh) if calls["n"] == 1: payload["open_questions"] = ["does dose matter?"] else: payload["answered_questions"] = ["does dose matter?"] payload["open_questions"] = [] return payload out = await iterative_evidence_convergence( TOPIC, min_iterations=3, max_iterations=10, search_fn=make_search({}), extract_fn=scripted) self.assertTrue(out["stopped_early"]) self.assertEqual(out["stop_reason"], "converged") self.assertEqual(out["answered_questions"], ["does dose matter?"]) self.assertEqual(out["unknown"], []) class TestGenericImprovements(unittest.IsolatedAsyncioTestCase): """Regression tests for the israel2.md advice (topic-agnostic).""" async def test_reliability_source_shape_caps(self): from blackcat.converge import reliability_tier, source_reliability self.assertEqual(source_reliability( "https://studentnews.harvard.edu/article"), 0.60) self.assertEqual(reliability_tier( "https://studentnews.harvard.edu/article"), "student_publication") self.assertEqual(source_reliability( "https://nytimes.com/opinion/essay"), 0.45) self.assertEqual(reliability_tier( "https://nytimes.com/opinion/essay"), "opinion") self.assertEqual(source_reliability( "https://repo.example.org/masterthesis.pdf"), 0.75) self.assertEqual(reliability_tier( "https://repo.example.org/masterthesis.pdf"), "thesis") self.assertEqual(source_reliability("https://cdc.gov/x"), 0.92) self.assertEqual(source_reliability("https://nature.com/a"), 0.80) async def test_counter_status_and_stop_gate(self): from blackcat.converge import EvidenceState, counter_evidence_status state = EvidenceState(topic=TOPIC) state.evidence_stances = {"E1@C1": "supporting"} self.assertEqual(counter_evidence_status(state), "not_searched") state.counter_rounds_attempted = 1 self.assertEqual(counter_evidence_status(state), "scarce_despite_counter_search") state.counter_rounds_attempted = 2 self.assertEqual(counter_evidence_status(state), "exhausted_no_counter_found") state.evidence_stances["E2@C1"] = "opposing" state.evidence_stances.update( {"E%d@C1" % i: "supporting" for i in range(3, 10)}) self.assertEqual(counter_evidence_status(state), "found") async def test_main_question_unresolved_when_claim_is_tangential(self): from blackcat.converge import ( Claim, EvidenceState, synthesize_conclusion, update_confidence_and_graph, ) state = EvidenceState( topic="who started the river valley conflict, group A or " "group B") side = Claim(id="C1", text="the river valley has a long " "history of trade") state.claims.append(side) item = EvidenceItem("E1", "https://a.org", "river valley trade " "history group", 0.8, 1.0, "2026-01-01", 0) state.evidence_items.append(item) state.claim_evidence_graph = {"C1": ["E1"]} state.evidence_stances = {"E1@C1": "supporting"} update_confidence_and_graph(state) out = synthesize_conclusion(state, stop_reason="max_iterations") self.assertFalse(out["main_question_resolved"]) self.assertTrue(out["conclusion"].startswith( "The main question remains unresolved")) self.assertEqual(out["verdict_status"], "incomplete_max_iterations") self.assertEqual(len(out["single_source_claims"]), 1) self.assertIn("single source", out["conclusion"]) async def test_one_sided_run_reports_counter_status(self): out = await iterative_evidence_convergence( TOPIC, min_iterations=1, max_iterations=2, early_stop=False, search_fn=make_search({}), extract_fn=extract) self.assertIn(out["counter_evidence_status"], ("scarce_despite_counter_search", "exhausted_no_counter_found", "found")) self.assertIn(out["verdict_status"], ("incomplete_max_iterations", "converged", "incomplete_dry_rounds")) class TestSectionD(unittest.IsolatedAsyncioTestCase): """Spec items (a)-(h) from the section-D research synthesis.""" async def test_a_anchor_frozen_and_drift_flagged(self): calls = {"n": 0} def shifting(topic, state, fresh): calls["n"] += 1 payload = extract(topic, state, fresh) payload["best_conclusion"] = \ "entirely different framing %d" % calls["n"] return payload out = await iterative_evidence_convergence( TOPIC, min_iterations=1, max_iterations=3, early_stop=False, search_fn=make_search({}), extract_fn=shifting) self.assertTrue(out["anchor_text"]) state = out["state"] self.assertEqual(state["anchor_text"], out["anchor_text"]) self.assertGreaterEqual(state["anchor_iteration"], 0) async def test_b_triple_shows_dilution_overall_kept(self): out = await iterative_evidence_convergence( TOPIC, min_iterations=1, max_iterations=2, early_stop=False, search_fn=make_search({}), extract_fn=extract) self.assertIn("confidence_headline", out) self.assertIn("confidence_coverage", out) self.assertEqual(out["confidence"], out["overall_confidence"]) async def test_c_exhausted_after_consecutive_zero_yield(self): from blackcat.converge import EvidenceState, counter_evidence_exhausted state = EvidenceState(topic=TOPIC) state.counter_rounds_attempted = 3 state.counter_rounds_with_opposing = 1 state.evidence_stances = {"E1@C1": "supporting"} self.assertFalse(counter_evidence_exhausted(state)) state.counter_consecutive_zero = 2 self.assertTrue(counter_evidence_exhausted(state)) async def test_d_distinct_domains_and_sole_primary(self): from blackcat.converge import Claim, EvidenceState, distinct_domains state = EvidenceState(topic=TOPIC) claim = Claim(id="C1", text=TOPIC) state.claims.append(claim) for i, url in enumerate(("https://a.org/1", "https://a.org/2", "https://b.org/1")): item = EvidenceItem("E%d" % i, url, TOPIC, 0.8, 1.0, None, 0) state.evidence_items.append(item) state.claim_evidence_graph.setdefault("C1", []).append( item.id) self.assertEqual(distinct_domains(state, claim), 2) async def test_e_tier_gap_and_primary_query(self): from blackcat.converge import Claim, EvidenceState, tier_gap state = EvidenceState(topic=TOPIC) claim = Claim(id="C1", text=TOPIC) state.claims.append(claim) item = EvidenceItem("E1", "https://randomblog.io/post", TOPIC, 0.3, 1.0, None, 0) state.evidence_items.append(item) state.claim_evidence_graph = {"C1": ["E1"]} state.evidence_stances = {"E1@C1": "supporting"} state.confidence_scores = {"C1": 0.9} self.assertTrue(tier_gap(state)["gap"]) async def test_f_facet_ledger_in_result(self): out = await iterative_evidence_convergence( TOPIC, min_iterations=1, max_iterations=1, search_fn=make_search({}), extract_fn=extract) self.assertGreaterEqual(out["facet_coverage"]["total"], 1) self.assertIn(out["facet_coverage"]["facets"][TOPIC], ("supported", "conflicted", "searched_none", "uncovered")) async def test_g_seeds(self): counter = {} out = await iterative_evidence_convergence( TOPIC, min_iterations=1, max_iterations=1, search_fn=make_search(counter), extract_fn=extract, seed_queries=["known author study on coffee focus"], seed_sources=[{"url": "https://nature.com/seeded", "title": "Seeded", "snippet": TOPIC + " seeded primary"}]) self.assertEqual(out["seeds_used"], {"queries": 1, "sources": 1}) self.assertTrue(any("seeded" in e["source"] for e in out["state"]["evidence_items"])) bad = await iterative_evidence_convergence( TOPIC, search_fn=make_search({}), seed_queries="nope") self.assertEqual(bad["code"], "bad_arguments") async def test_h_unicode_and_english_substitution(self): from blackcat.converge import ( _tokens, generate_initial_queries, is_non_english_topic, ) self.assertTrue(_tokens("Качество сна и кофе")) self.assertTrue(is_non_english_topic( "wie is de schuldige van het conflict")) queries = generate_initial_queries( "wie is de schuldige van het conflict", topic_en="who is responsible for the conflict") self.assertEqual(len(queries), 4) self.assertTrue(any("who is responsible" in q for q in queries)) english = generate_initial_queries(TOPIC, topic_en="ignored") self.assertTrue(any("experts" in q for q in english)) GATE_CLAIM = ( "The origins of the Israeli-Palestinian conflict lie in the struggle " "over land and labor markets on the frontier of Zionist settlement in " "late Ottoman Palestine, especially during the first and second " "aliyot (1882-1914).") TOPIC_NL = ("Wie is de oorspronkelijke veroorzaker / starter van het " "Israëlisch-Palestijnse (Arabisch-Israëlische) conflict? " "Historische oorsprong rond 1880-1948.") TOPIC_EN = ("Who originally started the Israeli-Palestinian conflict? " "Historical origins around 1880-1948.") class TestProposals(unittest.IsolatedAsyncioTestCase): """P1-P10 from converge_proposals.md (Gate 0 numbers: NL 0.0625, EN 0.40 against GATE_CLAIM, before the fix).""" async def test_gate0_relevance_with_normalised_scoring(self): from blackcat.converge import ( Claim, EvidenceState, claim_relevance_tier, ) raw = EvidenceState(topic=TOPIC_NL) claim = Claim(id="C1", text=GATE_CLAIM) self.assertEqual(claim_relevance_tier(raw, claim), 0.0) prepared = EvidenceState(topic=TOPIC_NL) prepared.scoring_topic = TOPIC_EN prepared.topic_keywords = ["Israeli-Palestinian conflict", "land", "labor markets", "Zionist settlement", "Ottoman"] self.assertGreaterEqual(claim_relevance_tier(prepared, claim), 0.5) unrelated = Claim(id="C2", text="Iron is the most common " "element in Earth's core") self.assertEqual(claim_relevance_tier(prepared, unrelated), 0.0) async def test_p2_counter_classification_strips_topic(self): from blackcat.converge import ( EvidenceState, _is_counter, generate_refinement_queries, ) self.assertFalse(_is_counter( "what is the evidence for why deterrence fails", "why deterrence fails")) self.assertTrue(_is_counter( "counterarguments against why deterrence fails", "why deterrence fails")) state = EvidenceState(topic="why deterrence fails") queries = generate_refinement_queries(state, 1) counter = [q for q in queries if _is_counter(q, state.topic)] self.assertGreaterEqual(len(counter) / len(queries), 0.30) async def test_p3_comparison_mode_divided_verdict(self): from blackcat.converge import ( Claim, EvidenceState, synthesize_conclusion, update_confidence_and_graph, ) state = EvidenceState(topic="who started the valley conflict") state.candidates = [ {"name": "Group A first move", "description": "A acted"}, {"name": "Group B first move", "description": "B acted"}] for i, (cid, text) in enumerate(( ("C1", "Group A first move started the valley conflict"), ("C2", "Group B first move started the valley conflict"))): claim = Claim(id=cid, text=text) state.claims.append(claim) item = EvidenceItem("E%d" % i, "https://d%d.org/x" % i, text, 0.8, 1.0, "2026-01-01", 0) state.evidence_items.append(item) state.claim_evidence_graph[cid] = [item.id] state.evidence_stances[item.id + "@" + cid] = "supporting" update_confidence_and_graph(state) out = synthesize_conclusion(state) self.assertTrue(out["comparison_mode"]) self.assertEqual(out["verdict_status"], "divided_no_dominant_candidate") self.assertEqual(len(out["candidates"]), 2) self.assertIn("literature divides", out["conclusion"]) async def test_p4_anchor_gate(self): from blackcat.converge import ( Claim, EvidenceState, synthesize_conclusion, update_confidence_and_graph, ) state = EvidenceState(topic="who started the valley conflict") state.anchor_claim_id = "C1" state.anchor_text = "unrelated baking recipes of the valley" claim = Claim(id="C1", text="unrelated baking recipes of the " "valley") state.claims.append(claim) item = EvidenceItem("E1", "https://a.org", "valley baking", 0.8, 1.0, "2026-01-01", 0) state.evidence_items.append(item) state.claim_evidence_graph = {"C1": ["E1"]} state.evidence_stances = {"E1@C1": "supporting"} state.current_best_conclusion = \ "who started the valley conflict group A" update_confidence_and_graph(state) out = synthesize_conclusion(state) self.assertEqual(out["verdict_status"], "answered_adjacent_question") self.assertEqual(out["adjacent_question"], state.anchor_text) self.assertIn("adjacent question", out["conclusion"]) async def test_p5_full_stance_set_and_sentence_boundary(self): out = await iterative_evidence_convergence( TOPIC, min_iterations=1, max_iterations=2, early_stop=False, search_fn=make_search({}), extract_fn=extract) self.assertIn("evidence_by_stance", out) self.assertTrue(out["conclusion"].rstrip().endswith( (".", ")", '"')) or out["conclusion"].endswith(".")) async def test_p6_ineffective_counter_search_not_exhausted(self): from blackcat.converge import EvidenceState, counter_evidence_status state = EvidenceState(topic=TOPIC) state.counter_rounds_attempted = 3 state.counter_query_relevances = [0.01, 0.02] state.evidence_stances = {"E1@C1": "supporting"} self.assertEqual(counter_evidence_status(state), "counter_search_ineffective") async def test_p7_facet_cap_and_topic_slot(self): out = await iterative_evidence_convergence( TOPIC, min_iterations=1, max_iterations=2, early_stop=False, search_fn=make_search({}), extract_fn=extract) self.assertLessEqual(out["facet_coverage"]["total"], 8) queries = [q for entry in out["iteration_history"] for q in entry["queries"]] self.assertIn(TOPIC, queries) async def test_p9_seeds_dropped_reported(self): out = await iterative_evidence_convergence( TOPIC, min_iterations=1, max_iterations=1, search_fn=make_search({}), extract_fn=extract, seed_queries=["s1 coffee", "s2 coffee", "s3 coffee", "s4 coffee", "s5 coffee", "s5 coffee"]) self.assertEqual(out["seeds_used"]["queries"], 4) reasons = {d["reason"] for d in out["seeds_dropped"]} self.assertIn("duplicate", reasons) self.assertTrue(any(r.startswith("round_cap") for r in reasons)) self.assertIn("run_fingerprint", out) self.assertTrue(out["run_fingerprint"].startswith("sha256:")) async def test_p10_cache_reuses_search(self): counter = {} cache = {} search = make_search(counter) await iterative_evidence_convergence( TOPIC, min_iterations=1, max_iterations=1, search_fn=search, extract_fn=extract, search_cache=cache) first = counter["calls"] await iterative_evidence_convergence( TOPIC, min_iterations=1, max_iterations=1, search_fn=search, extract_fn=extract, search_cache=cache) self.assertEqual(counter["calls"], first) class TestLoop(unittest.IsolatedAsyncioTestCase): async def test_early_stop_after_min_iterations(self): counter = {} out = await iterative_evidence_convergence( TOPIC, min_iterations=3, max_iterations=10, search_fn=make_search(counter), extract_fn=extract) self.assertTrue(out["ok"]) self.assertEqual(out["topic"], TOPIC) self.assertEqual(out["iterations"], 3) self.assertTrue(out["stopped_early"]) self.assertGreaterEqual(out["confidence"], 0.80) self.assertIn("Coffee improves focus", out["conclusion"]) self.assertTrue(out["supporting_evidence"]) self.assertEqual(out["unknown"], []) self.assertEqual(len(out["iteration_history"]), 3) state = out["state"] for field_name in ("claims", "evidence_items", "claim_evidence_graph", "confidence_scores", "contradictions", "open_questions", "source_diversity", "iteration_log", "current_best_conclusion", "overall_confidence"): self.assertIn(field_name, state) self.assertGreaterEqual(state["source_diversity"], 0.65) async def test_max_iterations_hard_cap_without_early_stop(self): out = await iterative_evidence_convergence( TOPIC, min_iterations=1, max_iterations=4, early_stop=False, search_fn=make_search({}), extract_fn=extract) self.assertEqual(out["iterations"], 4) self.assertFalse(out["stopped_early"]) async def test_fallback_extractor_without_llm(self): out = await iterative_evidence_convergence( TOPIC, min_iterations=1, max_iterations=2, search_fn=make_search({})) self.assertTrue(out["ok"]) self.assertTrue(out["claims"]) self.assertGreater(out["evidence_count"], 0) async def test_search_failure_degrades_not_crashes(self): def boom(query, count): raise RuntimeError("search down") out = await iterative_evidence_convergence( TOPIC, min_iterations=1, max_iterations=2, search_fn=boom) self.assertTrue(out["ok"]) self.assertEqual(out["evidence_count"], 0) self.assertIn("Insufficient evidence", out["conclusion"]) async def test_dedup_across_rounds(self): def same(query, count): return {"results": [{ "title": "One", "url": "https://nature.com/only", "date": "2026-01-01", "summary": "%s supports" % TOPIC}]} out = await iterative_evidence_convergence( TOPIC, min_iterations=1, max_iterations=3, early_stop=False, search_fn=same, extract_fn=extract) self.assertEqual(out["evidence_count"], 1) def _resp(message): return {"choices": [{"message": message, "finish_reason": "stop"}], "usage": {"prompt_tokens": 5}} class StubDeepSeek: """Scripted client: rsearch-shaped search + extractor chat. The ``search`` attribute doubles as the bot's ``_rsearch`` backend (the only search backend); ``chat`` serves the prompt loop, the prepare step, and extraction. """ model = "stub-model" def __init__(self): self.counter = {} self._sync_search = make_search(self.counter) self.main_calls = 0 async def search(self, query, count=10): return self._sync_search(query, count) async def chat(self, messages, **kwargs): system = messages[0].get("content", "") if messages else "" if "Prepare a research question" in system: return _resp({"role": "assistant", "content": json.dumps({ "language": "en", "normalized_question": TOPIC, "keywords": ["coffee", "focus", "caffeine"], "question_type": "factual", "candidates": []})}) if "extract structured evidence" in system: payload = json.loads(messages[1]["content"]) ids = [e["id"] for e in payload["new_evidence"]] return _resp({"role": "assistant", "content": json.dumps({ "claims": [{"text": "Coffee improves focus", "polarity": "supporting", "supporting_evidence_ids": ids}], "contradictions": [], "open_questions": [], "best_conclusion": "Coffee improves focus"})}) if kwargs.get("tools"): self.main_calls += 1 if self.main_calls == 1: return {"choices": [{"message": { "role": "assistant", "content": None, "tool_calls": [{ "id": "c1", "type": "function", "function": { "name": "iterative_evidence_convergence", "arguments": json.dumps({ "topic": TOPIC, "min_iterations": 1, "max_iterations": 2, "early_stop": False})}}]}, "finish_reason": "tool_calls"}], "usage": {"prompt_tokens": 5}} return _resp({"role": "assistant", "content": "Converged: coffee improves focus"}) return _resp({"role": "assistant", "content": "unused"}) class TestBotIntegration(unittest.IsolatedAsyncioTestCase): def setUp(self): self.tmp = tempfile.TemporaryDirectory() self.addCleanup(self.tmp.cleanup) def bot(self): bot = BlackCatBot(provider="deepseek", api_key="k", system="t", tools=FileTools(FileState(self.tmp.name))) bot._client = StubDeepSeek() bot._rsearch = bot._client self.addAsyncCleanup(bot.aclose) return bot async def test_run_tool_converges(self): bot = self.bot() out = await bot.run_tool("iterative_evidence_convergence", topic=TOPIC, min_iterations=1, max_iterations=2, early_stop=False) self.assertTrue(out["ok"]) self.assertEqual(out["iterations"], 2) self.assertIn("Coffee improves focus", out["conclusion"]) self.assertTrue(out["supporting_evidence"]) self.assertIn("evidence_count", out) self.assertNotIn("state", out) self.assertGreater(out["state_summary"]["evidence_items"], 0) self.assertTrue(out["english_scoring_used"]) async def test_alias_converge_on_claim(self): bot = self.bot() out = await bot.run_tool("converge_on_claim", topic=TOPIC, min_iterations=1, max_iterations=1) self.assertTrue(out["ok"]) async def test_bad_arguments_returned_not_raised(self): bot = self.bot() out = await bot.run_tool("iterative_evidence_convergence", topic="", min_iterations=1, max_iterations=1) self.assertEqual(out["code"], "bad_arguments") async def test_echo_provider_unavailable(self): async with BlackCatBot(provider="echo", system="t") as bot: out = await bot.run_tool("iterative_evidence_convergence", topic=TOPIC) self.assertEqual(out["code"], "unavailable") async def test_openrouter_provider_converges(self): bot = BlackCatBot(provider="openrouter", api_key="k", system="t", tools=FileTools(FileState(self.tmp.name))) bot._client = StubDeepSeek() bot._rsearch = bot._client self.addAsyncCleanup(bot.aclose) out = await bot.run_tool("iterative_evidence_convergence", topic=TOPIC, min_iterations=1, max_iterations=1) self.assertTrue(out["ok"]) async def test_prompt_loop_calls_converge_tool(self): bot = self.bot() reply = await bot.prompt("Is it true that %s?" % TOPIC) self.assertEqual(reply, "Converged: coffee improves focus") tool_msgs = [m for m in bot.messages if m.get("role") == "tool"] self.assertEqual(len(tool_msgs), 1) frames = [f for f in bot.transcript if f["kind"] == "tool"] self.assertTrue(frames[-1]["out"]["ok"]) self.assertEqual(frames[-1]["tool"], "iterative_evidence_convergence") self.assertIsNone(BlackCatBot.validate_wire(bot._wire_messages())) if __name__ == "__main__": unittest.main()