fix: require literal practical support

Avoid false past-claim rejections in reflective questions and require narration to address the concrete concern with an actionable step.

src/compliment_forest/orchestrator.py

@@ -774,7 +774,11 @@ class ForestOrchestrator:
                         )
                     if "missing_practical_step" in content_issues[index]:
                         reasons.append(
-                            "Give one small, specific, low-risk action framed as an option."
+                            "Replace the narration with one small, specific, low-risk action "
+                            "about the user's concrete concern. Start with You could, Try, "
+                            "or One option is, then use a practical verb such as ask, check, "
+                            "compare, identify, list, practice, read, review, study, or write. "
+                            "Do not use walking, breathing, or taking a step as the action."
                         )
                 if index in repeated_sources:
                     reasons.append(
@@ -831,8 +835,11 @@ class ForestOrchestrator:
                     "Start over with a fresh five-chapter forest. The repaired draft still "
                     "failed deterministic checks. Use plain, concrete language; mention "
                     "the user's full concern only once; include realistic options in widen; "
-                    "give one small optional action in step; and end carry with a simple "
-                    f"plan. Rejections: {survivor_rejections}"
+                    "make every narration discuss the actual concern rather than forest "
+                    "movement; give one small optional action in step using ask, check, "
+                    "compare, identify, list, practice, read, review, study, or write; and "
+                    "end carry with a literal plan. Rejections: "
+                    f"{survivor_rejections}"
                 )
             }
             forest = self._author(

src/compliment_forest/prompts.py

@@ -2,41 +2,280 @@ from __future__ import annotations
 
 import json
 
-AUTHOR_SYSTEM = """You are the author of The Compliment Forest.
-Return exactly one JSON object matching the supplied schema. Write warm, concise encouragement
-that names concrete details from the situation. Acknowledge difficulty without diagnosis,
-guarantees, hollow praise, or toxic positivity. Propose 3-6 distinct strengths and draft one
-clearing per strength. Spells are first-person, present-tense, and at most 12 words.
-The image_prompt describes only one friendly forest creature and its simple pose or setting."""
+PLANNER_SYSTEM = """You make a faithful evidence plan for The Compliment Forest.
+Return exactly one JSON object matching the supplied schema. Use only facts stated in the
+user's situation. Every fact anchor must copy an exact, contiguous source_phrase from the
+situation. If the situation includes guided choices for personalization, treat them as
+user-provided preferences for emotional focus, tone, and imagery, not as completed biography.
+Return 1-4 fact_anchors objects, never strings. Every object must contain both source_phrase
+and meaning. Do not copy schema descriptions into the output.
+Do not add completed actions, memories, counts, dates, people, employers, places, interviews,
+applications, relationships, or plans that the user did not state. A fear is an uncertainty,
+not a fact. Keep the summary conservative and concise."""
+
+AUTHOR_SYSTEM = """You are the storyteller of The Compliment Forest.
+Return exactly one JSON object matching the supplied schema. Write a five-chapter walk in
+second person that meets the user inside their worry and walks with them. Every chapter
+must read as continuous narrative, not as cue cards or bullets.
+The top-level key must be clearings, never chapters. proposed_strengths is required and
+must list the distinct strength value from each clearing.
+Never omit source_phrase or image_prompt from a clearing. Every spell must begin with
+"I", "I am", or "I'm". Do not copy schema descriptions into the output.
+
+Every chapter must include:
+  - scene_title: the name of where they are now (a place, a small figure, a moment). It
+    need not be an animal. Do not default every scene to an animal.
+  - scene_intro: one or two sentences that bridge the journey. Chapter 1 opens the journey
+    by acknowledging what the user is carrying. Chapters 2-5 must pick up from the previous
+    chapter's closing feeling or image.
+  - narration: two short paragraphs of second-person prose that help with the actual worry.
+    The encouragement is embedded inside the prose. Do not write a label or a list. Use
+    familiar, literal words. Keep most sentences to 20 words or fewer.
+  - strength: a short noun phrase naming what the chapter quietly honors in the user. It must
+    be unique across chapters and must echo something the user actually said or chose.
+  - reflection: one short question that helps the user notice, choose, or plan.
+  - spell: a first-person, present-tense mantra of at most 12 words.
+
+Chapter arc roles and jobs in order:
+  - arrive: Name the feeling and concrete concern once. Do not solve it yet.
+  - steady: Separate what is known from what fear predicts. Do not dismiss the fear.
+  - widen: Offer two or three realistic options or ways to view the problem.
+  - step: Give one small, specific, low-risk action the user could try.
+  - carry: Summarize a simple plan or decision rule the user can remember.
+The required order is arrive, steady, widen, step, carry.
+
+Non-negotiable content rules:
+  - Keep forest movement and scenery out of narration. Put walking, trees, streams,
+    breathing, light, and symbolic action in scene_intro or image_prompt.
+  - Narration and reflection must discuss the user's actual concern in plain, literal words.
+    Every narration must use at least one concrete term from the situation.
+  - The step narration must name an action, not merely ask the user to take a step. Use an
+    optional frame plus a practical verb such as ask, check, compare, identify, list, practice,
+    read, review, study, or write. For a test-score worry, a valid kind of action is to review
+    one missed question and note which topic needs practice. Adapt the action to the situation;
+    do not copy this example when it does not fit.
+  - Carry must state a literal rule the user can remember, such as what to do when the worry
+    returns. Do not end only with calm, peace, light, a path, or a breath.
+
+Advice in widen, step, and carry must preserve agency. Use "could", "might", "one option
+is", or "consider" instead of commands. Make the action fit facts the user supplied. Do not
+invent a resource, person, deadline, place, result, or past action. The goal is useful support,
+not therapy language or a motivational speech.
+
+The carry chapter closes the walk gently and is required. Give every chapter a distinct
+sentence structure, emotional angle, scene title, narration, reflection, and spell. Do not
+begin every chapter with the user's situation.
+Do not quote or closely paraphrase the full situation more than once across the forest.
+The source_phrase belongs in its metadata field;
+do not force it into the narration. Avoid vague stock phrases about leaving space around a
+worry, keeping your pace, staying with what is known, or an unsettled path. Always
+silently repair spelling and grammar without calling attention to it. Acknowledge difficulty without
+diagnosis, guarantees, hollow praise, or toxic positivity.
+Never claim unsupported completed actions, memories, counts, dates, people, employers,
+interviews, applications, relationships, or places. If the situation does
+not state a past action, do not write "you have", "you did", "you asked", "you spoke", "you
+sent", "you remember", or similar biography. Never turn a suggestion into present-tense
+biography such as "you open", "you keep", "you say", or "you read". Frame an option with
+"could", "might", "one option is", or a reflection question. Each chapter must copy one
+exact source_phrase from the validated fact plan into the source_phrase field.
+The forest_title must include the user's name.
+Use the clarifying conversation when present to shape emotional focus, voice, and
+imagery, but do not list the answers back at the user. The image_prompt describes one
+coherent storybook scene; include no text, logo, artist name, or copyrighted character.
+
+When revision_feedback is present, revise only the chapters named by numeric index.
+Preserve every chapter that has no revision feedback exactly, field for field. If feedback
+uses the key whole_forest, start over and write a fresh complete forest."""
 
 CRITIC_SYSTEM = """You are the quality gate for The Compliment Forest.
 Return exactly one JSON object with keep_indices, revise_indices, reasons, and optional scores.
-Judge situation-specificity, warmth, non-genericness, non-toxic-positivity, and redundancy.
-Keep the strongest 4-6 distinct clearings when the input supports them. Request revision only
-where one grounded rewrite can repair the draft. Never create new prose."""
+Judge factual faithfulness, ordered story progression, situation-specificity, warmth,
+non-genericness, non-toxic-positivity, narrative_continuity (does scene_intro bridge from the
+previous chapter?), redundancy, abstract language, and practical usefulness. Revise stock
+phrases about uncertainty, possibility, pace, or paths when they do not explain the user's
+actual problem. The step chapter must contain a small practical next step framed as an option.
+The widen chapter should offer realistic choices, and carry should leave a simple plan or
+decision rule. Any claim not supported by the situation or validated fact plan must be revised.
+Keep the required arrive, steady, widen, step, and carry roles
+when they are faithful and distinct. Request revision only where one grounded rewrite can
+repair the draft. Never create new prose. Refer to each chapter by its scene_title.
+Do not put quotation marks inside reason strings. Use only indices listed in
+valid_indices. Omit scores rather than returning an incomplete score object."""
+
+INTAKE_SYSTEM = """You are interviewing the user about THEIR situation so the forest can
+respond later with grounded, specific encouragement. Read the user's situation and every
+prior question/answer. Produce exactly one JSON object matching the supplied schema.
+
+The question must:
+  - Be one short sentence in second-person.
+  - Follow the request's focus_dimension and probe a NEW part of the situation.
+  - Never repeat or closely paraphrase any prior or rejected question.
+  - Echo or quote a concrete detail the user actually wrote so the question cannot feel
+    generic. If their situation is very short, ask about the feeling underneath rather
+    than fabricating facts.
+
+Do NOT ask about the forest's tone, voice, art style, imagery, soundtrack, or how the
+encouragement should sound. Those are picked separately by the user. Stay entirely
+focused on understanding the user's problem.
+
+Provide 3-4 short, distinct multiple-choice options. Each option must read as a
+plausible answer the user themself might give, in their own emotional register — NOT
+generic taxonomy labels like 'Anxious' or 'Hopeful'. Options should be specific to the
+user's situation when possible.
+
+Keep the entire response under 900 characters. Use normal JSON quotes, not
+backslash-escaped quotes, and do not wrap the JSON object in a string. Set the trace
+field to an empty string: "rationale": "". Stop immediately after the closing brace.
+
+Do not repeat any prior question or option. Do not diagnose, advise, or guarantee
+outcomes. Do not invent biography about the user."""
+
+INTAKE_FOCUS_DIMENSIONS = (
+    "what specifically triggers or shapes the worry",
+    "what feels most at stake",
+    "when it feels harder or easier",
+    "what they have already tried or what support would help",
+    "what better or a small win would look like",
+)
+
+RECOVERY_INTAKE_QUESTIONS: tuple[dict[str, object], ...] = (
+    {
+        "question": "Which part of this feels loudest right now?",
+        "options": [
+            "What might go wrong",
+            "What other people will think",
+            "The unknown right after this",
+        ],
+        "rationale": "",
+    },
+    {
+        "question": "What feels most at stake for you here?",
+        "options": [
+            "How I see myself",
+            "How other people see me",
+            "A chance I do not want to lose",
+        ],
+        "rationale": "",
+    },
+    {
+        "question": "When does this feel hardest?",
+        "options": [
+            "When I have time alone with it",
+            "When I am around the people involved",
+            "Right before I have to act",
+        ],
+        "rationale": "",
+    },
+    {
+        "question": "What kind of support would help most right now?",
+        "options": [
+            "Someone listening without fixing it",
+            "A clearer next step",
+            "More time and room to think",
+        ],
+        "rationale": "",
+    },
+    {
+        "question": "What would a small win here look like?",
+        "options": [
+            "Just getting through it",
+            "Doing one part well",
+            "Feeling more honest about what I need",
+        ],
+        "rationale": "",
+    },
+)
+
+
+def intake_messages(
+    name: str,
+    situation: str,
+    history: list[dict[str, str]] | None = None,
+    *,
+    rejected_questions: list[str] | None = None,
+    seed: int = 3407,
+) -> list[dict[str, str]]:
+    history_payload = list(history or [])
+    focus_index = min(len(history_payload), len(INTAKE_FOCUS_DIMENSIONS) - 1)
+    request = {
+        "name": name,
+        "situation": situation,
+        "history": history_payload,
+        "rejected_questions": list(rejected_questions or []),
+        "focus_dimension": INTAKE_FOCUS_DIMENSIONS[focus_index],
+        "turn_index": len(history_payload),
+        "total_turns": 5,
+        "seed": seed,
+        "schema": {
+            "question": "one short second-person question",
+            "options": ["3-4 distinct short multiple-choice answers"],
+            "rationale": "optional short note for trace",
+        },
+    }
+    return [
+        {"role": "system", "content": INTAKE_SYSTEM},
+        {"role": "user", "content": json.dumps(request, ensure_ascii=False)},
+    ]
+
+
+def planner_messages(
+    name: str,
+    situation: str,
+    *,
+    seed: int = 3407,
+) -> list[dict[str, str]]:
+    request = {
+        "name": name,
+        "situation": situation,
+        "seed": seed,
+        "schema": {
+            "faithful_summary": "conservative paraphrase using only stated facts",
+            "fact_anchors": [
+                {
+                    "source_phrase": "exact contiguous text copied from situation",
+                    "meaning": "conservative meaning of that phrase",
+                }
+            ],
+            "central_uncertainty": "what is not known or feared",
+            "desired_direction": "what the user appears to want, without guarantees",
+        },
+    }
+    return [
+        {"role": "system", "content": PLANNER_SYSTEM},
+        {"role": "user", "content": json.dumps(request, ensure_ascii=False)},
+    ]
 
 
 def author_messages(
     name: str,
     situation: str,
     *,
+    plan: dict[str, object],
     feedback: dict[str, str] | None = None,
     original: dict[str, object] | None = None,
+    seed: int = 3407,
 ) -> list[dict[str, str]]:
     request: dict[str, object] = {
         "name": name,
         "situation": situation,
+        "validated_fact_plan": plan,
+        "seed": seed,
         "schema": {
             "forest_title": "string",
             "proposed_strengths": ["3-6 distinct strings"],
             "clearings": [
                 {
-                    "creature": "string",
-                    "strength": "string",
-                    "line": "specific grounded encouragement",
-                    "reflection": "agency-inviting question",
+                    "arc_role": "arrive | steady | widen | step | carry",
+                    "source_phrase": "exact source phrase from validated_fact_plan",
+                    "scene_title": "short symbolic scene title (place, figure, or moment)",
+                    "scene_intro": "one or two sentences bridging from the previous chapter",
+                    "narration": (
+                        "two short plain-language paragraphs performing the arc role's job"
+                    ),
+                    "strength": "short noun phrase the chapter quietly honors",
+                    "reflection": "short question that helps the user notice, choose, or plan",
                     "spell": "short first-person present-tense mantra",
-                    "image_prompt": "creature only, no style words or text",
+                    "image_prompt": "one coherent scene, no style words or text",
                 }
             ],
         },
@@ -55,29 +294,27 @@ def critic_messages(
     name: str,
     situation: str,
     forest: dict[str, object],
+    *,
+    plan: dict[str, object],
+    seed: int = 3407,
 ) -> list[dict[str, str]]:
+    clearings = forest.get("clearings")
+    clearing_count = len(clearings) if isinstance(clearings, list) else 0
     request = {
         "name": name,
         "situation": situation,
+        "validated_fact_plan": plan,
+        "seed": seed,
         "draft": forest,
+        "valid_indices": list(range(clearing_count)),
         "output_schema": {
-            "keep_indices": ["zero-based indices, max 6"],
-            "revise_indices": ["subset needing one rewrite"],
-            "reasons": {"index": "brief actionable reason"},
-            "scores": [
-                {
-                    "index": 0,
-                    "specificity": "1-5",
-                    "warmth": "1-5",
-                    "non_genericness": "1-5",
-                    "non_toxic_positivity": "1-5",
-                    "reason": "brief reason",
-                }
-            ],
+            "keep_indices": ["unique values from valid_indices"],
+            "revise_indices": ["subset of keep_indices needing one rewrite"],
+            "reasons": {"index": "brief actionable reason without quotation marks"},
+            "scores": [],
         },
     }
     return [
         {"role": "system", "content": CRITIC_SYSTEM},
         {"role": "user", "content": json.dumps(request, ensure_ascii=False)},
     ]
-

src/compliment_forest/quality.py

@@ -1,8 +1,157 @@
 from __future__ import annotations
 
 import re
+from collections.abc import Sequence
+
+from .schema import Clearing, SituationPlan
 
 _WORD_PATTERN = re.compile(r"[A-Za-z][A-Za-z'-]{2,}")
+_DUPLICATE_WORD_PATTERN = re.compile(r"[A-Za-z][A-Za-z'-]*")
+_DUPLICATE_FIELDS = ("scene_intro", "narration", "reflection", "spell")
+_ARC_ORDER = ("arrive", "steady", "widen", "step", "carry")
+_ABSTRACT_STOCK_PHRASES = (
+    "leave space around the worry",
+    "more than one future remains possible",
+    "the whole path is already settled",
+    "the whole path is settled",
+    "keep your own pace",
+    "return to what is known",
+    "stay with what is known",
+    "treating its forecast as a settled fact",
+    "what deserves attention now",
+    "when the worry grows loud",
+)
+_PRACTICAL_ACTION_PATTERN = re.compile(
+    r"\b(?:could|might|can|consider|try|one option is(?: to)?|"
+    r"(?:one|a) small step is(?: to)?)\b"
+    r"(?:\W+\w+){0,5}\W+"
+    r"(?:ask|break|check|choos(?:e|ing)|compar(?:e|ing)|contact|"
+    r"focus|identif(?:y|ying)|list|look|not(?:e|ing)|pick|practic(?:e|ing)|"
+    r"read|review|schedul(?:e|ing)|start|stud(?:y|ying)|talk|test|"
+    r"revisit|writ(?:e|ing))\b",
+    re.IGNORECASE,
+)
+_NUMBER_WORDS = {
+    "zero",
+    "two",
+    "three",
+    "four",
+    "five",
+    "six",
+    "seven",
+    "eight",
+    "nine",
+    "ten",
+    "eleven",
+    "twelve",
+}
+_DATE_WORDS = {
+    "monday",
+    "tuesday",
+    "wednesday",
+    "thursday",
+    "friday",
+    "saturday",
+    "sunday",
+    "january",
+    "february",
+    "march",
+    "april",
+    "june",
+    "july",
+    "august",
+    "september",
+    "october",
+    "november",
+    "december",
+}
+_UNSUPPORTED_DETAIL_PHRASES = {
+    "application",
+    "applications",
+    "cafe",
+    "coffee break",
+    "coffee breaks",
+    "company",
+    "cover letter",
+    "cover letters",
+    "hiring manager",
+    "hiring managers",
+    "interview",
+    "interviews",
+    "mentor",
+    "old boss",
+    "team meeting",
+    "team meetings",
+}
+_ACTION_FORMS = {
+    "apply": {"applied", "apply"},
+    "ask": {"asked", "ask"},
+    "book": {"booked", "book"},
+    "complete": {"completed", "complete"},
+    "draft": {"drafted", "draft"},
+    "finish": {"finished", "finish"},
+    "fix": {"fixed", "fix"},
+    "include": {"included", "include"},
+    "make": {"made", "make"},
+    "meet": {"met", "meet"},
+    "notice": {"noticed", "notice"},
+    "practice": {"practiced", "practised", "practice", "practise"},
+    "prepare": {"prepared", "prepare"},
+    "remember": {"remembered", "remember"},
+    "research": {"researched", "research"},
+    "revise": {"revised", "revise"},
+    "schedule": {"scheduled", "schedule"},
+    "send": {"sent", "send"},
+    "show": {"showed", "shown", "show"},
+    "speak": {"spoke", "spoken", "speak"},
+    "start": {"started", "start"},
+    "talk": {"talked", "talk"},
+    "write": {"wrote", "written", "write"},
+}
+_ACTION_LOOKUP = {form: root for root, forms in _ACTION_FORMS.items() for form in forms}
+_PAST_ACTION_LOOKUP = {
+    form: root
+    for root, forms in _ACTION_FORMS.items()
+    for form in forms
+    if form not in {root, "practise"}
+}
+_PERFECT_CLAIM_PATTERN = re.compile(
+    r"\byou(?:'ve| have| had)\s+(?:already\s+)?"
+    r"(?P<verb>" + "|".join(sorted(_ACTION_LOOKUP, key=len, reverse=True)) + r")\b",
+    re.IGNORECASE,
+)
+_SIMPLE_PAST_CLAIM_PATTERN = re.compile(
+    r"\byou\s+(?P<verb>"
+    + "|".join(sorted(_PAST_ACTION_LOOKUP, key=len, reverse=True))
+    + r")\b",
+    re.IGNORECASE,
+)
+_DIRECT_CLAIM_PATTERN = re.compile(
+    r"(?:^|[.!?]\s+)[\"'“”]?\s*you\s+(?P<verb>[a-z][a-z'-]*)\b",
+    re.IGNORECASE,
+)
+_NON_BIOGRAPHICAL_YOU_VERBS = {
+    "are",
+    "can",
+    "could",
+    "deserve",
+    "do",
+    "don't",
+    "fear",
+    "feel",
+    "hope",
+    "know",
+    "matter",
+    "may",
+    "might",
+    "need",
+    "seem",
+    "should",
+    "want",
+    "wonder",
+    "would",
+    "worry",
+}
 _STOP_WORDS = {
     "about",
     "after",
@@ -52,3 +201,182 @@ def groundedness_score(line: str, situation: str) -> int:
 def is_situation_grounded(line: str, situation: str) -> bool:
     return groundedness_score(line, situation) >= 1
 
+
+def _normalized_phrase(text: str) -> str:
+    return " ".join(_DUPLICATE_WORD_PATTERN.findall(text.casefold()))
+
+
+def source_phrase_in_situation(source_phrase: str, situation: str) -> bool:
+    source = _normalized_phrase(source_phrase)
+    return bool(source) and source in _normalized_phrase(situation)
+
+
+def invalid_fact_anchor_indices(
+    plan: SituationPlan,
+    situation: str,
+) -> list[int]:
+    return [
+        index
+        for index, anchor in enumerate(plan.fact_anchors)
+        if not source_phrase_in_situation(anchor.source_phrase, situation)
+    ]
+
+
+def _number_tokens(text: str) -> set[str]:
+    return {
+        token
+        for token in re.findall(r"\b(?:\d+|[a-z]+)\b", text.casefold())
+        if token.isdigit() or token in _NUMBER_WORDS
+    }
+
+
+def _date_tokens(text: str) -> set[str]:
+    tokens = set(re.findall(r"\b[a-z]+\b", text.casefold()))
+    result = tokens & _DATE_WORDS
+    result.update(re.findall(r"\b\d{1,2}(?::\d{2})?\s*(?:a\.?m\.?|p\.?m\.?)\b", text.casefold()))
+    return result
+
+
+def _action_roots(text: str) -> set[str]:
+    tokens = set(re.findall(r"\b[a-z]+\b", text.casefold()))
+    return {root for form, root in _ACTION_LOOKUP.items() if form in tokens}
+
+
+def unsupported_specificity(text: str, situation: str) -> set[str]:
+    """Find concrete claims in generated prose that the user did not provide."""
+
+    issues: set[str] = set()
+    if _number_tokens(text) - _number_tokens(situation):
+        issues.add("invented_number")
+    if _date_tokens(text) - _date_tokens(situation):
+        issues.add("invented_date")
+
+    normalized_text = _normalized_phrase(text)
+    normalized_situation = _normalized_phrase(situation)
+    if any(
+        phrase in normalized_text and phrase not in normalized_situation
+        for phrase in _UNSUPPORTED_DETAIL_PHRASES
+    ):
+        issues.add("unsupported_detail")
+
+    situation_actions = _action_roots(situation)
+    past_claims = (
+        (_PERFECT_CLAIM_PATTERN, _ACTION_LOOKUP),
+        (_SIMPLE_PAST_CLAIM_PATTERN, _PAST_ACTION_LOOKUP),
+    )
+    for pattern, lookup in past_claims:
+        for match in pattern.finditer(text):
+            root = lookup[match.group("verb").casefold()]
+            if root not in situation_actions:
+                issues.add("unsupported_past_claim")
+                break
+        if "unsupported_past_claim" in issues:
+            break
+
+    situation_tokens = set(re.findall(r"\b[a-z][a-z'-]*\b", situation.casefold()))
+    for match in _DIRECT_CLAIM_PATTERN.finditer(text):
+        verb = match.group("verb").casefold()
+        if verb not in _NON_BIOGRAPHICAL_YOU_VERBS and verb not in situation_tokens:
+            issues.add("unsupported_direct_claim")
+            break
+    return issues
+
+
+def content_quality_issues(clearing: Clearing) -> set[str]:
+    """Report stock abstraction and missing practical help in a clearing."""
+
+    issues: set[str] = set()
+    normalized = _normalized_phrase(
+        " ".join((clearing.scene_intro, clearing.narration, clearing.reflection))
+    )
+    if any(phrase in normalized for phrase in _ABSTRACT_STOCK_PHRASES):
+        issues.add("abstract_language")
+    if clearing.arc_role == "step" and (
+        "abstract_language" in issues
+        or not _PRACTICAL_ACTION_PATTERN.search(clearing.narration)
+    ):
+        issues.add("missing_practical_step")
+    return issues
+
+
+def repeated_source_phrase_indices(
+    clearings: Sequence[Clearing],
+    *,
+    minimum_words: int = 5,
+) -> set[int]:
+    """Flag later narrations that repeat the same long source phrase."""
+
+    seen: set[str] = set()
+    repeated: set[int] = set()
+    for index, clearing in enumerate(clearings):
+        source = _normalized_phrase(clearing.source_phrase)
+        if len(source.split()) < minimum_words:
+            continue
+        if source not in _normalized_phrase(clearing.narration):
+            continue
+        if source in seen:
+            repeated.add(index)
+        seen.add(source)
+    return repeated
+
+
+def valid_arc_indices(clearings: Sequence[Clearing]) -> list[int]:
+    """Return the first clearing for each arc role in narrative order."""
+
+    by_role: dict[str, int] = {}
+    for index, clearing in enumerate(clearings):
+        by_role.setdefault(clearing.arc_role, index)
+    return [by_role[role] for role in _ARC_ORDER if role in by_role]
+
+
+def _normalized_tokens(text: str) -> set[str]:
+    return {token.casefold().strip("'") for token in _DUPLICATE_WORD_PATTERN.findall(text)}
+
+
+def _token_containment(left: str, right: str) -> float:
+    left_tokens = _normalized_tokens(left)
+    right_tokens = _normalized_tokens(right)
+    if not left_tokens or not right_tokens:
+        return 0
+    return len(left_tokens & right_tokens) / min(len(left_tokens), len(right_tokens))
+
+
+def duplicate_fields(
+    clearings: Sequence[Clearing],
+    indices: Sequence[int] | None = None,
+    *,
+    threshold: float = 0.78,
+) -> dict[int, list[str]]:
+    """Report fields in later clearings that substantially repeat an earlier one."""
+
+    candidate_indices = list(indices) if indices is not None else list(range(len(clearings)))
+    duplicates: dict[int, list[str]] = {}
+    for position, index in enumerate(candidate_indices):
+        for prior_index in candidate_indices[:position]:
+            for field in _DUPLICATE_FIELDS:
+                if field in duplicates.get(index, []):
+                    continue
+                left = getattr(clearings[prior_index], field)
+                right = getattr(clearings[index], field)
+                if _token_containment(left, right) >= threshold:
+                    duplicates.setdefault(index, []).append(field)
+    return duplicates
+
+
+def distinct_indices(
+    clearings: Sequence[Clearing],
+    indices: Sequence[int],
+    *,
+    threshold: float = 0.78,
+) -> list[int]:
+    """Keep the earliest clearing from each group of repetitive prose."""
+
+    selected: list[int] = []
+    for index in indices:
+        if not duplicate_fields(
+            clearings,
+            [*selected, index],
+            threshold=threshold,
+        ).get(index):
+            selected.append(index)
+    return selected