Update app.py

app.py

@@ -320,7 +320,9 @@ def clean_value(value: Any, max_chars: int = 220) -> str:
         text = ", ".join(clean_value(item, max_chars=max_chars) for item in value)
     else:
         text = str(value)
-    text = re.sub(r"\s+", " ", text).strip()
+    text = text.replace("\\r\\n", "\n").replace("\\n", "\n").replace("\\t", " ")
+    text = re.sub(r"[ \t]+", " ", text)
+    text = re.sub(r"\n{3,}", "\n\n", text).strip()
     if not text or text.lower() in {"none", "null", "nan", "not provided"}:
         return ""
     if text.startswith(("http://", "https://")):
@@ -533,6 +535,14 @@ def sentence_join(lines: list[str]) -> str:
     return "\n".join(line for line in lines if line)
 
 
+def normalize_answer_text(text: str) -> str:
+    text = str(text or "").replace("\\r\\n", "\n").replace("\\n", "\n").replace("\\t", " ")
+    text = re.sub(r"[ \t]+", " ", text)
+    text = re.sub(r" *\n *", "\n", text)
+    text = re.sub(r"\n{3,}", "\n\n", text)
+    return text.strip()
+
+
 def build_grounded_answer(req: AnswerRequest, lang: str) -> str:
     rows = req.context or req.rows or []
     if not rows:
@@ -618,7 +628,7 @@ def simplify_grounded_answer(generator: IdemGenerator, req: AnswerRequest, lang:
         simplified = first_reformulation(raw_simplified, grounded_answer, lang)
     if not simplified:
         simplified = grounded_answer
-    simplified = clean_value(simplified, max_chars=3000)
+    simplified = normalize_answer_text(clean_value(simplified, max_chars=3000))
     if introduces_unsupported_ownership(grounded_answer, simplified):
         simplified = grounded_answer
     if not simplified or not text_matches_language(simplified, lang):
@@ -646,7 +656,7 @@ def clean_idem_output(raw: str, original_input: str) -> str:
         if cleaned:
             lines.append(cleaned)
 
-    return "\n".join(lines).strip() or raw.strip()
+    return normalize_answer_text("\n".join(lines).strip() or raw.strip())
 
 
 def first_reformulation(output_text: str, original_text: str, lang: str) -> str:
@@ -736,4 +746,4 @@ def answer(req: AnswerRequest):
     if not result:
         result = FALLBACKS[lang]
 
-    return {"answer": result, "source": source}
+    return {"answer": normalize_answer_text(result), "source": source}