Spaces:
Sleeping
Sleeping
deploy(S4): Blender headless pipeline + WebAR client + backend fixes
Browse files- backend/events/omega_event_bus.py +21 -15
- backend/main.py +31 -1
- backend/routes/engine_routes.py +310 -0
- backend/routes/mobile_bridge_routes.py +389 -1
- backend/routes/persona_routes.py +41 -7
- backend/routes/token_task_routes.py +94 -0
- backend/services/token_manager.py +720 -669
- backend/services/usb_monitor.py +22 -2
- backend/tools/web_search_tools.py +27 -6
- backend/voice/engines/kokoro_engine.py +0 -0
- backend/voice/jarvis_live.py +202 -0
- backend/voice/tts.py +12 -0
- config.py +72 -2
- modules/alert_manager.py +307 -0
- modules/assistant_identity.py +409 -1
- modules/deferred_intent.py +268 -0
- modules/executor.py +12 -1
- modules/initiative.py +612 -0
- modules/intent.py +203 -0
- modules/omega_executor.py +667 -0
- modules/process_utils.py +44 -0
- modules/security_review.py +12 -1
- modules/task_authoring.py +360 -0
backend/events/omega_event_bus.py
CHANGED
|
@@ -62,23 +62,29 @@ async def publish_omega_event(event: OmegaEvent):
|
|
| 62 |
# 3. routes through existing §0.4 voice (stubbed for now if kokoro_engine not fully connected here,
|
| 63 |
# but requested by user spec to synthesize_tts)
|
| 64 |
if event_type_warrants_speech(event.event_type):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
try:
|
| 66 |
-
from
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
# Context stub if ResponseContext does not exist natively
|
| 70 |
-
class StubContext:
|
| 71 |
-
type = "automation_status"
|
| 72 |
-
|
| 73 |
-
await synthesize_tts(
|
| 74 |
-
describe_event_for_speech(event),
|
| 75 |
-
persona=persona,
|
| 76 |
-
context=StubContext()
|
| 77 |
-
)
|
| 78 |
-
except ImportError as e:
|
| 79 |
-
logging.warning(f"Could not import TTS engine for omega event speech: {e}")
|
| 80 |
except Exception as e:
|
| 81 |
-
logging.error(f"Error
|
| 82 |
|
| 83 |
# 4. Trigger vault auto-refresh
|
| 84 |
try:
|
|
|
|
| 62 |
# 3. routes through existing §0.4 voice (stubbed for now if kokoro_engine not fully connected here,
|
| 63 |
# but requested by user spec to synthesize_tts)
|
| 64 |
if event_type_warrants_speech(event.event_type):
|
| 65 |
+
# SPEAK the event, rather than synthesising it and dropping it.
|
| 66 |
+
#
|
| 67 |
+
# This called synthesize_kokoro directly and discarded the return value.
|
| 68 |
+
# synthesize_kokoro returns WAV bytes; it does not play anything and has
|
| 69 |
+
# no audio device. So every "feature implemented", "bug found",
|
| 70 |
+
# "game won" and "research alert" ran the full voice pipeline — model
|
| 71 |
+
# load, RVC conversion, the whole DSP chain — and then let the audio go
|
| 72 |
+
# out of scope unheard. Seconds of GPU work per event, for silence.
|
| 73 |
+
#
|
| 74 |
+
# It also ignored `persona`: the parameter was passed, but
|
| 75 |
+
# synthesize_kokoro takes **kwargs and only reads `profile`, so a FRIDAY
|
| 76 |
+
# event would have been rendered in the JARVIS voice had anyone heard it.
|
| 77 |
+
#
|
| 78 |
+
# core.voice.speak() is the correct destination. It picks the voice from
|
| 79 |
+
# the active persona, and its single-owner IPC means this backend
|
| 80 |
+
# process hands the line to whichever process owns the speakers instead
|
| 81 |
+
# of loading a second copy of the models.
|
| 82 |
try:
|
| 83 |
+
from core.voice import speak
|
| 84 |
+
speak(describe_event_for_speech(event), event="omega_event",
|
| 85 |
+
cooldown_s=4.0)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 86 |
except Exception as e:
|
| 87 |
+
logging.error(f"Error speaking omega event: {e}")
|
| 88 |
|
| 89 |
# 4. Trigger vault auto-refresh
|
| 90 |
try:
|
backend/main.py
CHANGED
|
@@ -113,6 +113,8 @@ from backend.routes.easter_egg_routes import router as easter_egg_router
|
|
| 113 |
from backend.routes.omega_routes import router as omega_router
|
| 114 |
from backend.routes.persona_routes import router as persona_router
|
| 115 |
from backend.routes.internet_routes import router as internet_router
|
|
|
|
|
|
|
| 116 |
from backend.routes.sentinel_routes import router as sentinel_router
|
| 117 |
from backend.routers.gaming_routes import router as gaming_router
|
| 118 |
from backend.routes.model_proxy_routes import router as model_proxy_router # AR-FIX-SESSION
|
|
@@ -206,6 +208,17 @@ app.include_router(distribution_router, prefix="")
|
|
| 206 |
app.include_router(family_device_router, prefix="")
|
| 207 |
app.include_router(persona_router, prefix="/persona", dependencies=[Depends(verify_token)])
|
| 208 |
app.include_router(internet_router, prefix="/internet", dependencies=[Depends(verify_token)])
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 209 |
app.include_router(sentinel_router, prefix="/sentinel", dependencies=[Depends(verify_token)])
|
| 210 |
app.include_router(gaming_router, prefix="", dependencies=[Depends(verify_token)]) # gaming routes already have /gaming prefix
|
| 211 |
app.include_router(model_proxy_router, prefix="/api") # AR-FIX-SESSION: model provider proxy stubs (no auth — WebAR calls these from browser)
|
|
@@ -690,7 +703,24 @@ def main():
|
|
| 690 |
asyncio.run(run_all_migrations(db_path))
|
| 691 |
|
| 692 |
cloud_env = os.environ.get("CLOUD_ENV", "false").lower() == "true"
|
| 693 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 694 |
base_port = int(os.environ.get("PORT", "7860")) if cloud_env else int(os.environ.get("JARVIS_PORT", "7474"))
|
| 695 |
|
| 696 |
for port in range(base_port, base_port + 11):
|
|
|
|
| 113 |
from backend.routes.omega_routes import router as omega_router
|
| 114 |
from backend.routes.persona_routes import router as persona_router
|
| 115 |
from backend.routes.internet_routes import router as internet_router
|
| 116 |
+
from backend.routes.token_task_routes import router as token_task_router
|
| 117 |
+
from backend.routes.engine_routes import router as engine_router
|
| 118 |
from backend.routes.sentinel_routes import router as sentinel_router
|
| 119 |
from backend.routers.gaming_routes import router as gaming_router
|
| 120 |
from backend.routes.model_proxy_routes import router as model_proxy_router # AR-FIX-SESSION
|
|
|
|
| 208 |
app.include_router(family_device_router, prefix="")
|
| 209 |
app.include_router(persona_router, prefix="/persona", dependencies=[Depends(verify_token)])
|
| 210 |
app.include_router(internet_router, prefix="/internet", dependencies=[Depends(verify_token)])
|
| 211 |
+
# The token manager, reachable by the phone. Lets the handset use the full
|
| 212 |
+
# 37-key domain routing without carrying a single key itself.
|
| 213 |
+
app.include_router(token_task_router, prefix="/tokens", dependencies=[Depends(verify_token)])
|
| 214 |
+
# The full engine — unlimited tasks, intent, initiative, deferred orders,
|
| 215 |
+
# silence. One implementation, called by the phone rather than duplicated in
|
| 216 |
+
# Kotlin, so the handset can never drift from how JARVIS actually behaves.
|
| 217 |
+
#
|
| 218 |
+
# /engine, NOT /omega: omega_routes.py is the long-standing Autonomous Research
|
| 219 |
+
# and Enhancement API (research, propose, approve, upgrade) and already owns
|
| 220 |
+
# that prefix via its own APIRouter(prefix="/omega").
|
| 221 |
+
app.include_router(engine_router, prefix="/engine", dependencies=[Depends(verify_token)])
|
| 222 |
app.include_router(sentinel_router, prefix="/sentinel", dependencies=[Depends(verify_token)])
|
| 223 |
app.include_router(gaming_router, prefix="", dependencies=[Depends(verify_token)]) # gaming routes already have /gaming prefix
|
| 224 |
app.include_router(model_proxy_router, prefix="/api") # AR-FIX-SESSION: model provider proxy stubs (no auth — WebAR calls these from browser)
|
|
|
|
| 703 |
asyncio.run(run_all_migrations(db_path))
|
| 704 |
|
| 705 |
cloud_env = os.environ.get("CLOUD_ENV", "false").lower() == "true"
|
| 706 |
+
|
| 707 |
+
# Bind the LAN, not just loopback.
|
| 708 |
+
#
|
| 709 |
+
# This was "0.0.0.0" only in cloud mode and "127.0.0.1" everywhere else, so
|
| 710 |
+
# on the desktop the bridge answered nothing but the machine it ran on. The
|
| 711 |
+
# phone's whole LAN path — the `http://<pc>:7474/api/...` half of the dual
|
| 712 |
+
# -path contract the Guardian app is built around — could never connect, and
|
| 713 |
+
# the JARVIS voice in particular cannot come from anywhere else: the cloud
|
| 714 |
+
# Space carries no RVC, so the PC is the only host that can render it.
|
| 715 |
+
#
|
| 716 |
+
# This is not an authentication hole. auth_middleware waives the token only
|
| 717 |
+
# for `request.client.host in ("127.0.0.1", "localhost", "::1")`; every LAN
|
| 718 |
+
# request must present a bearer token that passes verify_master_token, and
|
| 719 |
+
# that check is unchanged. What changes is only whether the socket is
|
| 720 |
+
# reachable at all.
|
| 721 |
+
#
|
| 722 |
+
# JARVIS_BIND_HOST forces it back to loopback on an untrusted network.
|
| 723 |
+
host = os.environ.get("JARVIS_BIND_HOST") or "0.0.0.0"
|
| 724 |
base_port = int(os.environ.get("PORT", "7860")) if cloud_env else int(os.environ.get("JARVIS_PORT", "7474"))
|
| 725 |
|
| 726 |
for port in range(base_port, base_port + 11):
|
backend/routes/engine_routes.py
ADDED
|
@@ -0,0 +1,310 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""backend/routes/engine_routes.py — the whole engine, reachable from anywhere.
|
| 2 |
+
|
| 3 |
+
WHY A ROUTE AND NOT A KOTLIN PORT
|
| 4 |
+
---------------------------------
|
| 5 |
+
Everything built this session is Python: modules/omega_executor.py plans and
|
| 6 |
+
runs unlimited tasks (writing its own code when nothing fits),
|
| 7 |
+
modules/intent.py understands what was meant rather than matching words,
|
| 8 |
+
modules/deferred_intent.py deliberates irreversible orders, and
|
| 9 |
+
modules/initiative.py acts unprompted.
|
| 10 |
+
|
| 11 |
+
Re-implementing those four in Kotlin would give the phone a SECOND definition
|
| 12 |
+
of how JARVIS thinks and acts — which is exactly the mistake
|
| 13 |
+
MobileJarvisEngine.kt already made with the persona text, and exactly how
|
| 14 |
+
core/brain.py drifted into telling JARVIS he was FRIDAY on every turn. Two
|
| 15 |
+
copies of a brain diverge.
|
| 16 |
+
|
| 17 |
+
So the phone calls this. One engine, one behaviour, and anything added to the
|
| 18 |
+
Python is instantly available on the handset with no rebuild.
|
| 19 |
+
|
| 20 |
+
WHERE THE WORK HAPPENS
|
| 21 |
+
----------------------
|
| 22 |
+
On the machine running this backend. That is the point: "shut down the PC",
|
| 23 |
+
"finish that download", "close every window" are PC actions, and the phone is
|
| 24 |
+
the remote. Requests that are really about the handset itself (torch, phone
|
| 25 |
+
volume) still belong to the on-device engine, and this does not replace that.
|
| 26 |
+
"""
|
| 27 |
+
|
| 28 |
+
from typing import Optional
|
| 29 |
+
|
| 30 |
+
from fastapi import APIRouter, HTTPException
|
| 31 |
+
from pydantic import BaseModel
|
| 32 |
+
|
| 33 |
+
router = APIRouter()
|
| 34 |
+
|
| 35 |
+
|
| 36 |
+
class ExecuteRequest(BaseModel):
|
| 37 |
+
request: str
|
| 38 |
+
context: str = ""
|
| 39 |
+
# Off by default over the network: an irreversible action asked for from a
|
| 40 |
+
# phone in a pocket deserves a deliberate opt-in, not a default.
|
| 41 |
+
allow_destructive: bool = False
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
class PlanRequest(BaseModel):
|
| 45 |
+
request: str
|
| 46 |
+
context: str = ""
|
| 47 |
+
|
| 48 |
+
|
| 49 |
+
class IntentRequest(BaseModel):
|
| 50 |
+
text: str
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
@router.post("/execute")
|
| 54 |
+
async def omega_execute(req: ExecuteRequest) -> dict:
|
| 55 |
+
"""Do whatever was asked — the full unlimited engine.
|
| 56 |
+
|
| 57 |
+
Chains, waits, real commands, and model-authored code for anything with no
|
| 58 |
+
existing handler. Identical to what the desktop voice path runs.
|
| 59 |
+
"""
|
| 60 |
+
text = (req.request or "").strip()
|
| 61 |
+
if not text:
|
| 62 |
+
raise HTTPException(status_code=400, detail="request is required")
|
| 63 |
+
try:
|
| 64 |
+
from modules.omega_executor import execute
|
| 65 |
+
spoken: list[str] = []
|
| 66 |
+
res = execute(
|
| 67 |
+
text,
|
| 68 |
+
speak=spoken.append,
|
| 69 |
+
context=req.context or "",
|
| 70 |
+
allow_destructive=bool(req.allow_destructive),
|
| 71 |
+
)
|
| 72 |
+
return {
|
| 73 |
+
"status": "ok",
|
| 74 |
+
"ok": res.ok,
|
| 75 |
+
"spoken": res.spoken,
|
| 76 |
+
"said": spoken,
|
| 77 |
+
"cancelled": res.cancelled,
|
| 78 |
+
"steps": [
|
| 79 |
+
{"index": s.index, "kind": s.kind, "summary": s.summary,
|
| 80 |
+
"ok": s.ok, "output": s.output[:1200], "error": s.error[:600]}
|
| 81 |
+
for s in res.steps
|
| 82 |
+
],
|
| 83 |
+
}
|
| 84 |
+
except Exception as e:
|
| 85 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 86 |
+
|
| 87 |
+
|
| 88 |
+
@router.post("/plan")
|
| 89 |
+
async def omega_plan(req: PlanRequest) -> dict:
|
| 90 |
+
"""Plan without doing. Lets the phone show what WOULD happen first."""
|
| 91 |
+
text = (req.request or "").strip()
|
| 92 |
+
if not text:
|
| 93 |
+
raise HTTPException(status_code=400, detail="request is required")
|
| 94 |
+
try:
|
| 95 |
+
from modules.omega_executor import plan_request, is_destructive
|
| 96 |
+
plan = plan_request(text, req.context or "")
|
| 97 |
+
steps = plan.get("steps") or []
|
| 98 |
+
return {
|
| 99 |
+
"status": "ok",
|
| 100 |
+
"say": plan.get("say", ""),
|
| 101 |
+
"degraded": bool(plan.get("degraded")),
|
| 102 |
+
"step_count": len(steps),
|
| 103 |
+
"steps": steps,
|
| 104 |
+
"has_irreversible": any(
|
| 105 |
+
is_destructive(str(s.get("cmd") or s.get("code") or "")
|
| 106 |
+
+ " " + str(s.get("name") or ""))
|
| 107 |
+
for s in steps
|
| 108 |
+
),
|
| 109 |
+
}
|
| 110 |
+
except Exception as e:
|
| 111 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
@router.post("/intent")
|
| 115 |
+
async def omega_intent(req: IntentRequest) -> dict:
|
| 116 |
+
"""What did they MEAN? act / converse / wake / mute / unmute / cancel."""
|
| 117 |
+
try:
|
| 118 |
+
from modules.intent import classify
|
| 119 |
+
got = classify(req.text or "")
|
| 120 |
+
return {
|
| 121 |
+
"status": "ok",
|
| 122 |
+
"intent": got.kind,
|
| 123 |
+
"confidence": got.confidence,
|
| 124 |
+
"payload": got.payload,
|
| 125 |
+
"source": got.source,
|
| 126 |
+
}
|
| 127 |
+
except Exception as e:
|
| 128 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
@router.get("/repairs")
|
| 132 |
+
async def list_repairs() -> dict:
|
| 133 |
+
"""Commands that had to be corrected at runtime.
|
| 134 |
+
|
| 135 |
+
The immediate repair unblocks the operator; this is the evidence trail for
|
| 136 |
+
the lasting fix. A command that needs correcting every single run is a bug
|
| 137 |
+
that self-repair is otherwise hiding very effectively.
|
| 138 |
+
"""
|
| 139 |
+
try:
|
| 140 |
+
from modules.omega_executor import pending_repairs
|
| 141 |
+
items = pending_repairs()
|
| 142 |
+
return {"status": "ok", "count": len(items), "repairs": items[-25:]}
|
| 143 |
+
except Exception as e:
|
| 144 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
@router.get("/depth_model_url")
|
| 148 |
+
async def depth_model_url() -> dict:
|
| 149 |
+
"""Where the phone should fetch the monocular-depth weights.
|
| 150 |
+
|
| 151 |
+
Served rather than hardcoded in the APK because a guessed URL that fails
|
| 152 |
+
leaves 2D->3D silently off forever. Three candidates were tested live and
|
| 153 |
+
all three failed (401 on a private path, 404 on two public mirrors), which
|
| 154 |
+
is exactly why this is configuration and not a constant.
|
| 155 |
+
|
| 156 |
+
Set OMEGA_DEPTH_MODEL_URL to a reachable .onnx and every handset picks it
|
| 157 |
+
up with no rebuild. Until then this returns empty and the phone reports
|
| 158 |
+
"no model URL configured" instead of pretending to be a 3D viewer.
|
| 159 |
+
"""
|
| 160 |
+
import os
|
| 161 |
+
|
| 162 |
+
# Depth Anything V2 small, int8. Chosen because:
|
| 163 |
+
# * the repo and file were CONFIRMED to exist via the HF API
|
| 164 |
+
# (onnx-community/depth-anything-v2-small -> onnx/model_int8.onnx);
|
| 165 |
+
# * int8 keeps it small enough to download and run on a handset;
|
| 166 |
+
# * V2 small is markedly better than MiDaS small on indoor scenes, which
|
| 167 |
+
# is what a photo or video in a virtual theatre usually is.
|
| 168 |
+
#
|
| 169 |
+
# NOT verified by downloading: this environment cannot follow Hugging Face's
|
| 170 |
+
# CDN redirect, so the bytes were never fetched here. The path is real; the
|
| 171 |
+
# first phone to run it is the real test.
|
| 172 |
+
DEFAULT = ("https://huggingface.co/onnx-community/depth-anything-v2-small"
|
| 173 |
+
"/resolve/main/onnx/model_int8.onnx")
|
| 174 |
+
|
| 175 |
+
url = (os.environ.get("OMEGA_DEPTH_MODEL_URL") or DEFAULT).strip()
|
| 176 |
+
|
| 177 |
+
# Input size travels WITH the url. Getting these out of step gives a model
|
| 178 |
+
# that loads happily and returns nonsense depth, which shows up as a
|
| 179 |
+
# headache rather than an error — far worse than a clean failure.
|
| 180 |
+
try:
|
| 181 |
+
size = int(os.environ.get("OMEGA_DEPTH_INPUT_SIZE") or 0)
|
| 182 |
+
except ValueError:
|
| 183 |
+
size = 0
|
| 184 |
+
if not size:
|
| 185 |
+
size = 256 if "midas" in url.lower() else 518
|
| 186 |
+
|
| 187 |
+
return {
|
| 188 |
+
"status": "ok",
|
| 189 |
+
"url": url,
|
| 190 |
+
"input_size": size,
|
| 191 |
+
"configured": True,
|
| 192 |
+
"is_default": url == DEFAULT,
|
| 193 |
+
"note": ("Default Depth Anything V2 small (int8). Override with "
|
| 194 |
+
"OMEGA_DEPTH_MODEL_URL, and set OMEGA_DEPTH_INPUT_SIZE to "
|
| 195 |
+
"match that model's expected input."),
|
| 196 |
+
}
|
| 197 |
+
|
| 198 |
+
|
| 199 |
+
@router.get("/capabilities")
|
| 200 |
+
async def omega_capabilities() -> dict:
|
| 201 |
+
"""Every power currently granted. Honest about what it can actually do."""
|
| 202 |
+
try:
|
| 203 |
+
from modules.omega_executor import capability_catalog
|
| 204 |
+
caps = capability_catalog()
|
| 205 |
+
return {"status": "ok", "count": len(caps), "capabilities": caps}
|
| 206 |
+
except Exception as e:
|
| 207 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
# ── Unprompted action, from the phone's point of view ───────────────────────
|
| 211 |
+
@router.get("/initiative")
|
| 212 |
+
async def initiative_state() -> dict:
|
| 213 |
+
"""What he noticed, and anything he did while nobody was watching."""
|
| 214 |
+
try:
|
| 215 |
+
from modules.initiative import observe, pending_report, format_report, is_running
|
| 216 |
+
return {
|
| 217 |
+
"status": "ok",
|
| 218 |
+
"running": is_running(),
|
| 219 |
+
"situation": observe(),
|
| 220 |
+
"pending_report": pending_report(),
|
| 221 |
+
"spoken_report": format_report(),
|
| 222 |
+
}
|
| 223 |
+
except Exception as e:
|
| 224 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 225 |
+
|
| 226 |
+
|
| 227 |
+
@router.post("/initiative/tick")
|
| 228 |
+
async def initiative_tick() -> dict:
|
| 229 |
+
"""Look now rather than waiting for the timer."""
|
| 230 |
+
try:
|
| 231 |
+
from modules.initiative import tick
|
| 232 |
+
spoken: list[str] = []
|
| 233 |
+
v = tick(speak=spoken.append)
|
| 234 |
+
if v is None:
|
| 235 |
+
return {"status": "ok", "silenced": True}
|
| 236 |
+
return {
|
| 237 |
+
"status": "ok",
|
| 238 |
+
"act": v.act, "say": v.say, "action": v.action,
|
| 239 |
+
"urgency": v.urgency, "topic": v.topic, "said": spoken,
|
| 240 |
+
}
|
| 241 |
+
except Exception as e:
|
| 242 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
@router.post("/initiative/acknowledge")
|
| 246 |
+
async def initiative_ack() -> dict:
|
| 247 |
+
"""The operator has seen the report; stop offering it."""
|
| 248 |
+
try:
|
| 249 |
+
from modules.initiative import mark_reported, note_user_contact
|
| 250 |
+
note_user_contact()
|
| 251 |
+
mark_reported()
|
| 252 |
+
return {"status": "ok"}
|
| 253 |
+
except Exception as e:
|
| 254 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
# ── Held irreversible orders ────────────────────────────────────────────────
|
| 258 |
+
@router.get("/deferred")
|
| 259 |
+
async def deferred_list() -> dict:
|
| 260 |
+
"""Orders held back, why, and when they will be reconsidered."""
|
| 261 |
+
try:
|
| 262 |
+
from modules.deferred_intent import pending
|
| 263 |
+
return {"status": "ok", "orders": pending()}
|
| 264 |
+
except Exception as e:
|
| 265 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 266 |
+
|
| 267 |
+
|
| 268 |
+
class WithdrawRequest(BaseModel):
|
| 269 |
+
order: Optional[str] = None
|
| 270 |
+
|
| 271 |
+
|
| 272 |
+
@router.post("/deferred/withdraw")
|
| 273 |
+
async def deferred_withdraw(req: WithdrawRequest) -> dict:
|
| 274 |
+
"""Changed their mind. Drops one order, or all of them."""
|
| 275 |
+
try:
|
| 276 |
+
from modules.deferred_intent import withdraw
|
| 277 |
+
withdraw(req.order)
|
| 278 |
+
return {"status": "ok"}
|
| 279 |
+
except Exception as e:
|
| 280 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 281 |
+
|
| 282 |
+
|
| 283 |
+
# ── Silence ─────────────────────────────────────────────────────────────────
|
| 284 |
+
class SilenceRequest(BaseModel):
|
| 285 |
+
silent: bool
|
| 286 |
+
reason: str = ""
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
@router.post("/silence")
|
| 290 |
+
async def set_silence(req: SilenceRequest) -> dict:
|
| 291 |
+
"""Total mute, from the phone. No timeout — released only explicitly."""
|
| 292 |
+
try:
|
| 293 |
+
from modules.alert_manager import silence_all, unsilence_all, is_silenced
|
| 294 |
+
if req.silent:
|
| 295 |
+
silence_all(req.reason or "asked from phone")
|
| 296 |
+
else:
|
| 297 |
+
unsilence_all()
|
| 298 |
+
return {"status": "ok", "silenced": is_silenced()}
|
| 299 |
+
except Exception as e:
|
| 300 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 301 |
+
|
| 302 |
+
|
| 303 |
+
@router.get("/silence")
|
| 304 |
+
async def get_silence() -> dict:
|
| 305 |
+
try:
|
| 306 |
+
from modules.alert_manager import is_silenced, silence_info
|
| 307 |
+
return {"status": "ok", "silenced": is_silenced(), "info": silence_info()}
|
| 308 |
+
except Exception as e:
|
| 309 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 310 |
+
|
backend/routes/mobile_bridge_routes.py
CHANGED
|
@@ -209,6 +209,15 @@ async def link_status() -> dict[str, Any]:
|
|
| 209 |
"detail": detail,
|
| 210 |
"queued_commands": queued,
|
| 211 |
"pc_status_age_seconds": age,
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 212 |
"pc_status": status,
|
| 213 |
}
|
| 214 |
|
|
@@ -357,6 +366,351 @@ def _gemini_envelope(text: str) -> dict[str, Any]:
|
|
| 357 |
}
|
| 358 |
|
| 359 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 360 |
@router.post("/chat")
|
| 361 |
async def chat(request: Request) -> dict[str, Any]:
|
| 362 |
"""Synchronous LLM turn for the mobile app.
|
|
@@ -372,6 +726,15 @@ async def chat(request: Request) -> dict[str, Any]:
|
|
| 372 |
This runs the prompt synchronously through the token manager's full
|
| 373 |
Gemini→NVIDIA fallback chain and returns the Gemini envelope, so both callers
|
| 374 |
work with no APK change.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 375 |
"""
|
| 376 |
body = await _json_body(request)
|
| 377 |
text = str(body.get("text") or "").strip()
|
|
@@ -381,9 +744,34 @@ async def chat(request: Request) -> dict[str, Any]:
|
|
| 381 |
persona = "jarvis"
|
| 382 |
if not text:
|
| 383 |
return _gemini_envelope("Standing by.")
|
|
|
|
|
|
|
|
|
|
| 384 |
try:
|
| 385 |
from backend.services.token_manager import gemini_call_with_checkpoint
|
| 386 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 387 |
return _gemini_envelope(reply or "")
|
| 388 |
except Exception as exc:
|
| 389 |
log.warning("mobile /api/chat LLM call failed: %s", exc)
|
|
|
|
| 209 |
"detail": detail,
|
| 210 |
"queued_commands": queued,
|
| 211 |
"pc_status_age_seconds": age,
|
| 212 |
+
# Where the phone can reach the PC directly for VOICE.
|
| 213 |
+
#
|
| 214 |
+
# Commands need no address — the cloud relays them — but the voice
|
| 215 |
+
# cannot work that way: this Space has no RVC and no model weights, so
|
| 216 |
+
# only the PC can render it. Rather than make the user type an IP, the
|
| 217 |
+
# desktop advertises its own reachable addresses in its periodic status
|
| 218 |
+
# and they are handed straight through here, on the call the app already
|
| 219 |
+
# polls. Tailscale first, then LAN. Empty until a PC has reported.
|
| 220 |
+
"voice_hosts": status.get("voice_hosts") or [],
|
| 221 |
"pc_status": status,
|
| 222 |
}
|
| 223 |
|
|
|
|
| 366 |
}
|
| 367 |
|
| 368 |
|
| 369 |
+
async def _measure_live_state(context: dict[str, Any]) -> dict[str, Any]:
|
| 370 |
+
"""The facts of this moment, measured rather than assumed.
|
| 371 |
+
|
| 372 |
+
Asked "is the PC connected?", the assistant used to answer from nothing at
|
| 373 |
+
all — the chat path sent the characterisation and the user's sentence and no
|
| 374 |
+
system state whatsoever — so it produced a confident "connected, sir" about a
|
| 375 |
+
machine that was switched off. Everything below is read live; anything that
|
| 376 |
+
cannot be read is reported as unknown rather than omitted, because a stated
|
| 377 |
+
unknown is far harder to confabulate over than a silent gap.
|
| 378 |
+
"""
|
| 379 |
+
state: dict[str, Any] = {}
|
| 380 |
+
|
| 381 |
+
# PC link — the authoritative answer is whether a desktop is holding an
|
| 382 |
+
# outbound socket right now, not whether one was ever paired.
|
| 383 |
+
try:
|
| 384 |
+
from backend.ws.agent_ws import ws_manager
|
| 385 |
+
pcs = list(getattr(ws_manager, "pc_connections", []) or [])
|
| 386 |
+
state["PC connected right now"] = "yes" if pcs else "no - the PC is offline"
|
| 387 |
+
state["PC sockets held"] = len(pcs)
|
| 388 |
+
except Exception as exc:
|
| 389 |
+
log.debug("live state: pc link unreadable: %s", exc)
|
| 390 |
+
state["PC connected right now"] = "unknown - could not read the link"
|
| 391 |
+
|
| 392 |
+
try:
|
| 393 |
+
import datetime
|
| 394 |
+
state["current time"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M %Z").strip()
|
| 395 |
+
except Exception:
|
| 396 |
+
pass
|
| 397 |
+
|
| 398 |
+
for key, label in (
|
| 399 |
+
("device_name", "operator device"),
|
| 400 |
+
("surface", "which surface this came from"),
|
| 401 |
+
("location", "operator location"),
|
| 402 |
+
):
|
| 403 |
+
val = context.get(key)
|
| 404 |
+
if val:
|
| 405 |
+
state[label] = str(val)[:120]
|
| 406 |
+
state.setdefault("which surface this came from", "the phone app")
|
| 407 |
+
|
| 408 |
+
# The handset's own senses — where the operator is, what they are doing, who
|
| 409 |
+
# is around, whether speech would be overheard. SituationSensor.kt gathers
|
| 410 |
+
# these; anything it could not read is simply absent, never invented.
|
| 411 |
+
#
|
| 412 |
+
# This is the difference between an assistant that adapts to the situations
|
| 413 |
+
# somebody thought to enumerate and one that adapts to the situation actually
|
| 414 |
+
# occurring. The readings are raw on purpose: "ringer: vibrate, calendar
|
| 415 |
+
# event: standup, headphones: no" is evidence he reasons over, not a label
|
| 416 |
+
# like "in a meeting" decided for him by a table on the phone.
|
| 417 |
+
senses = context.get("senses")
|
| 418 |
+
if isinstance(senses, dict):
|
| 419 |
+
for key, val in senses.items():
|
| 420 |
+
if val not in (None, ""):
|
| 421 |
+
state[str(key)[:60]] = str(val)[:160]
|
| 422 |
+
|
| 423 |
+
# Held irreversible orders and unprompted work, when this process has them.
|
| 424 |
+
try:
|
| 425 |
+
from modules.deferred_intent import pending as _deferred_pending
|
| 426 |
+
held = _deferred_pending() or []
|
| 427 |
+
if held:
|
| 428 |
+
state["orders held pending deliberation"] = "; ".join(
|
| 429 |
+
str(o.get("order", ""))[:80] for o in held[:4]
|
| 430 |
+
)
|
| 431 |
+
except Exception:
|
| 432 |
+
pass
|
| 433 |
+
try:
|
| 434 |
+
from modules.initiative import pending_report as _init_pending
|
| 435 |
+
done = _init_pending() or []
|
| 436 |
+
if done:
|
| 437 |
+
state["things done unprompted, not yet reported"] = len(done)
|
| 438 |
+
except Exception:
|
| 439 |
+
pass
|
| 440 |
+
|
| 441 |
+
return state
|
| 442 |
+
|
| 443 |
+
|
| 444 |
+
# Separate budgets, because they used to share one.
|
| 445 |
+
#
|
| 446 |
+
# Three rounds total meant a single PC run plus a single search exhausted the
|
| 447 |
+
# turn, and the operator got the canned "I couldn't get to a straight answer"
|
| 448 |
+
# even though nothing had failed. Real research is: search, read what came back,
|
| 449 |
+
# search again for the specific thing. That is three searches minimum before a
|
| 450 |
+
# word is said.
|
| 451 |
+
_MAX_ROUNDS = 7 # hard stop on the whole turn
|
| 452 |
+
_MAX_SEARCHES = 4 # web lookups per turn
|
| 453 |
+
_MAX_PC_RUNS = 2 # desktop executions per turn
|
| 454 |
+
|
| 455 |
+
|
| 456 |
+
async def _run_on_pc(task: str) -> str:
|
| 457 |
+
"""Run an arbitrary task on the operator's desktop and report what happened.
|
| 458 |
+
|
| 459 |
+
Two routes, in order of directness. On the Space there is no desktop to run
|
| 460 |
+
on, so the work goes down the socket the PC holds open (the same path
|
| 461 |
+
/max_autonomy/task uses). Running in-process is correct when this backend IS
|
| 462 |
+
the desktop, which is how the exe serves it.
|
| 463 |
+
|
| 464 |
+
Irreversible steps are deliberated over and held with a cancel window inside
|
| 465 |
+
the executor itself, so nothing here needs to second-guess the request.
|
| 466 |
+
"""
|
| 467 |
+
if not task:
|
| 468 |
+
return "No task given."
|
| 469 |
+
|
| 470 |
+
# In-process: this backend is the machine.
|
| 471 |
+
try:
|
| 472 |
+
from modules.omega_executor import execute as _execute
|
| 473 |
+
import asyncio as _asyncio
|
| 474 |
+
result = await _asyncio.to_thread(_execute, task)
|
| 475 |
+
steps = getattr(result, "steps", None) or []
|
| 476 |
+
lines = [f"{'ok' if getattr(s, 'ok', False) else 'FAILED'}: "
|
| 477 |
+
f"{getattr(s, 'summary', '')}"[:180] for s in steps[:12]]
|
| 478 |
+
spoken = getattr(result, "spoken", "") or ""
|
| 479 |
+
return (spoken + "\n" + "\n".join(lines)).strip() or "Completed with no output."
|
| 480 |
+
except ImportError:
|
| 481 |
+
pass
|
| 482 |
+
except Exception as exc:
|
| 483 |
+
log.warning("in-process executor failed: %s", exc)
|
| 484 |
+
return f"The engine errored: {str(exc)[:300]}"
|
| 485 |
+
|
| 486 |
+
# Cloud: relay to whichever desktop is holding a socket open.
|
| 487 |
+
try:
|
| 488 |
+
from backend.ws.agent_ws import ws_manager
|
| 489 |
+
if not getattr(ws_manager, "pc_connections", None):
|
| 490 |
+
return ("The PC is offline, so there was nothing to run it on. "
|
| 491 |
+
"It was not queued.")
|
| 492 |
+
outcome = await ws_manager.execute_on_pcs(task)
|
| 493 |
+
if not outcome.get("delivered"):
|
| 494 |
+
return "The PC did not accept the task."
|
| 495 |
+
if outcome.get("timed_out"):
|
| 496 |
+
return "The PC took the task and is still working on it."
|
| 497 |
+
said = outcome.get("spoken") or []
|
| 498 |
+
return "\n".join(str(s) for s in said)[:1500] or "Done, with no output reported."
|
| 499 |
+
except Exception as exc:
|
| 500 |
+
log.warning("PC relay failed: %s", exc)
|
| 501 |
+
return f"Could not reach the PC: {str(exc)[:200]}"
|
| 502 |
+
|
| 503 |
+
# Readings that describe the operator's circumstances rather than the machine's.
|
| 504 |
+
# Only these go in the preamble; battery and PC-link facts belong in LIVE STATE
|
| 505 |
+
# where they can be quoted, not in the line that shapes how he answers.
|
| 506 |
+
_CIRCUMSTANCE_KEYS = (
|
| 507 |
+
"local time", "part of day", "day", "roughly where", "moving",
|
| 508 |
+
"app in use", "screen", "in a calendar event right now", "event location",
|
| 509 |
+
"bluetooth connected", "ringer", "media playing",
|
| 510 |
+
"speaking aloud would be overheard", "wifi network",
|
| 511 |
+
)
|
| 512 |
+
|
| 513 |
+
|
| 514 |
+
def _situated(text: str, live: dict[str, Any]) -> str:
|
| 515 |
+
"""Put the operator's circumstances immediately before their question.
|
| 516 |
+
|
| 517 |
+
LIVE STATE already carries these, at the end of a system prompt that runs to
|
| 518 |
+
a hundred and fifty lines of characterisation. That is enough for him to
|
| 519 |
+
QUOTE a reading when asked, and demonstrably not enough for him to ACT on
|
| 520 |
+
one: given 68 km/h and a car's bluetooth, he cheerfully began reciting a
|
| 521 |
+
multi-step server setup to a man at the wheel.
|
| 522 |
+
|
| 523 |
+
Attention follows recency far more than depth, so the same facts are
|
| 524 |
+
restated here, adjacent to the question, as the thing to think about first.
|
| 525 |
+
Same readings, no new claims — only a position that gets them used.
|
| 526 |
+
"""
|
| 527 |
+
bits = [f"{k}: {v}" for k, v in live.items()
|
| 528 |
+
if k in _CIRCUMSTANCE_KEYS and v not in (None, "", "unknown")]
|
| 529 |
+
if not bits:
|
| 530 |
+
return text
|
| 531 |
+
return (
|
| 532 |
+
"RIGHT NOW, around the operator: " + "; ".join(bits) + ".\n"
|
| 533 |
+
"Work out what that means about what they are doing, what they can "
|
| 534 |
+
"actually use this second, and who can hear.\n"
|
| 535 |
+
"\n"
|
| 536 |
+
"DECIDE THE SHAPE OF YOUR ANSWER BEFORE YOU WRITE IT, NOT AFTER.\n"
|
| 537 |
+
"\n"
|
| 538 |
+
"If the circumstances mean they cannot use a long answer right now, then "
|
| 539 |
+
"the short answer IS the answer. Do not deliver the long one and then "
|
| 540 |
+
"remark that they cannot use it. Giving a man at the wheel four steps of "
|
| 541 |
+
"server configuration and closing with 'of course, you are driving' is "
|
| 542 |
+
"not judgement — it is the whole procedure plus an apology for it, and "
|
| 543 |
+
"he could not act on a word of it. What he needed was the one-line "
|
| 544 |
+
"answer and an offer to walk him through it when he is somewhere he can "
|
| 545 |
+
"type. Same for anything overheard: if it should not be said aloud here, "
|
| 546 |
+
"do not say it and then note that it was sensitive.\n"
|
| 547 |
+
"\n"
|
| 548 |
+
"Judge every time from the readings themselves — these are examples of "
|
| 549 |
+
"the reasoning, not a list of cases to match. Do not mention the "
|
| 550 |
+
"readings unless they are what was asked about.\n\n"
|
| 551 |
+
f"THEY SAID: {text}"
|
| 552 |
+
)
|
| 553 |
+
|
| 554 |
+
|
| 555 |
+
def _history_block(history: list[Any]) -> str:
|
| 556 |
+
"""The last few turns, as plain transcript.
|
| 557 |
+
|
| 558 |
+
Without this the endpoint is amnesiac: every POST carries one utterance and
|
| 559 |
+
no antecedent, so "download it" has no *it*. The handset has kept a six-turn
|
| 560 |
+
window for a long time — it was simply never sent (see MobileJarvisEngine,
|
| 561 |
+
where the request carrying it was built and then dropped on the floor).
|
| 562 |
+
|
| 563 |
+
A pronoun with nothing to bind to is the single most common way this
|
| 564 |
+
assistant looked stupid, and it was never a reasoning failure.
|
| 565 |
+
"""
|
| 566 |
+
rows = []
|
| 567 |
+
for turn in history[-8:]:
|
| 568 |
+
if isinstance(turn, dict):
|
| 569 |
+
role = str(turn.get("role") or "user").strip().lower()
|
| 570 |
+
said = str(turn.get("text") or "").strip()
|
| 571 |
+
elif isinstance(turn, (list, tuple)) and len(turn) == 2:
|
| 572 |
+
role, said = str(turn[0]).strip().lower(), str(turn[1]).strip()
|
| 573 |
+
else:
|
| 574 |
+
continue
|
| 575 |
+
if not said:
|
| 576 |
+
continue
|
| 577 |
+
who = "THEY" if role in ("user", "operator", "human") else "YOU"
|
| 578 |
+
rows.append(f"{who}: {said[:600]}")
|
| 579 |
+
if not rows:
|
| 580 |
+
return ""
|
| 581 |
+
return (
|
| 582 |
+
"THE CONVERSATION SO FAR (oldest first). Pronouns in the new message — "
|
| 583 |
+
"'it', 'that', 'her', 'the same one' — almost always refer to something "
|
| 584 |
+
"here. Resolve them from this before doing anything else:\n"
|
| 585 |
+
+ "\n".join(rows) + "\n\n"
|
| 586 |
+
)
|
| 587 |
+
|
| 588 |
+
|
| 589 |
+
async def _run_lookup_loop(
|
| 590 |
+
text: str,
|
| 591 |
+
persona: str,
|
| 592 |
+
live: dict[str, Any],
|
| 593 |
+
history: list[Any] | None = None,
|
| 594 |
+
) -> str:
|
| 595 |
+
"""Ask, and honour a SEARCH: line by actually searching, then ask again.
|
| 596 |
+
|
| 597 |
+
This is what makes "who is <a real person>" answerable. Previously chat was a
|
| 598 |
+
bare model call with no way to reach the internet, so any question about
|
| 599 |
+
someone the model did not already know came back as "I can't find them" —
|
| 600 |
+
without a single lookup having been attempted. search_web already existed and
|
| 601 |
+
was wired to research and sentinel, never to conversation.
|
| 602 |
+
|
| 603 |
+
The decision to search is the model's, per turn, rather than a list of
|
| 604 |
+
question shapes that mean "search now" — a list would fail on the first shape
|
| 605 |
+
nobody thought of, which is the exact failure being fixed.
|
| 606 |
+
"""
|
| 607 |
+
from backend.services.token_manager import gemini_call_with_checkpoint
|
| 608 |
+
|
| 609 |
+
# Everything learned this turn, oldest first, carried into every later round.
|
| 610 |
+
#
|
| 611 |
+
# This used to be a single `prompt` variable that each round OVERWROTE, so a
|
| 612 |
+
# second search silently erased the first one's results. Any request needing
|
| 613 |
+
# two steps — find the official page, then find the file on it — could not
|
| 614 |
+
# work: by the time he had the second answer he had lost the first. That is
|
| 615 |
+
# research reduced to one lucky query.
|
| 616 |
+
evidence: list[str] = []
|
| 617 |
+
asked = _situated(text, live)
|
| 618 |
+
if history:
|
| 619 |
+
asked = _history_block(history) + asked
|
| 620 |
+
|
| 621 |
+
def _compose() -> str:
|
| 622 |
+
if not evidence:
|
| 623 |
+
return asked
|
| 624 |
+
return (
|
| 625 |
+
"WHAT YOU HAVE ACTUALLY DONE AND FOUND SO FAR THIS TURN, in order:\n\n"
|
| 626 |
+
+ "\n\n".join(evidence)
|
| 627 |
+
+ "\n\nUse it. If it answers them, answer now and name what you found "
|
| 628 |
+
"— the real title, the real link, the real number. If it does not, "
|
| 629 |
+
"either search once more for the missing piece or tell them plainly "
|
| 630 |
+
"that you could not find it.\n\n"
|
| 631 |
+
"NEVER report something as done that you did not do, and never "
|
| 632 |
+
"describe a result you did not receive. 'I couldn't find it' is a "
|
| 633 |
+
"good answer. A confident invention is not.\n\n"
|
| 634 |
+
+ asked
|
| 635 |
+
)
|
| 636 |
+
|
| 637 |
+
searches = 0
|
| 638 |
+
pc_runs = 0
|
| 639 |
+
for _ in range(_MAX_ROUNDS):
|
| 640 |
+
reply = (await gemini_call_with_checkpoint(
|
| 641 |
+
_compose(), task_type="general", persona=persona, live_state=live,
|
| 642 |
+
can_search=(searches < _MAX_SEARCHES), can_act=True, can_read=True,
|
| 643 |
+
) or "").strip()
|
| 644 |
+
|
| 645 |
+
# READ is fulfilled on the PHONE, not here — only the handset can see its
|
| 646 |
+
# own screen, notifications, messages and clipboard. So a READ: reply is
|
| 647 |
+
# handed back verbatim; MobileJarvisEngine on the device performs the read
|
| 648 |
+
# and re-POSTs with the result. The cloud has no eyes on the phone and
|
| 649 |
+
# must not pretend to.
|
| 650 |
+
if reply.upper().startswith("READ:"):
|
| 651 |
+
return reply
|
| 652 |
+
|
| 653 |
+
# Hand the whole thing to the unlimited engine on the desktop.
|
| 654 |
+
#
|
| 655 |
+
# This is the escape hatch that makes "anything, including things nobody
|
| 656 |
+
# coded" true rather than aspirational: omega_executor plans a request
|
| 657 |
+
# into steps, runs shell, and writes fresh Python when no existing tool
|
| 658 |
+
# fits, repairing a failed step and retrying. The chat path had no idea
|
| 659 |
+
# it existed, so the assistant refused work his own machine could do.
|
| 660 |
+
if reply.upper().startswith("PC:"):
|
| 661 |
+
task = reply.split(":", 1)[1].strip()
|
| 662 |
+
if pc_runs >= _MAX_PC_RUNS:
|
| 663 |
+
evidence.append(
|
| 664 |
+
f"You wanted the desktop to: {task}\nYou have already used the "
|
| 665 |
+
"desktop as many times as this turn allows. Work with what you "
|
| 666 |
+
"have, or say what is still missing.")
|
| 667 |
+
continue
|
| 668 |
+
pc_runs += 1
|
| 669 |
+
outcome = await _run_on_pc(task)
|
| 670 |
+
evidence.append(f"YOU RAN ON THE DESKTOP: {task}\nWHAT HAPPENED:\n{outcome}")
|
| 671 |
+
continue
|
| 672 |
+
|
| 673 |
+
if not reply.upper().startswith("SEARCH:"):
|
| 674 |
+
return reply
|
| 675 |
+
|
| 676 |
+
query = reply.split(":", 1)[1].strip()
|
| 677 |
+
if not query:
|
| 678 |
+
return reply
|
| 679 |
+
searches += 1
|
| 680 |
+
log.info("chat: assistant requested a lookup: %s", query[:120])
|
| 681 |
+
try:
|
| 682 |
+
from backend.tools.web_search_tools import search_web
|
| 683 |
+
found = await search_web(query, num_results=5)
|
| 684 |
+
except Exception as exc:
|
| 685 |
+
log.warning("chat lookup failed: %s", exc)
|
| 686 |
+
found = {"error": str(exc)[:200]}
|
| 687 |
+
|
| 688 |
+
if found.get("error"):
|
| 689 |
+
block = f"It failed: {found['error']}. You did NOT get any results."
|
| 690 |
+
else:
|
| 691 |
+
rows = found.get("results") or []
|
| 692 |
+
if not rows:
|
| 693 |
+
block = "It returned nothing. You did NOT find anything."
|
| 694 |
+
else:
|
| 695 |
+
block = "\n".join(
|
| 696 |
+
f"- {r.get('title', '')}: {str(r.get('snippet') or r.get('body') or '')[:300]}"
|
| 697 |
+
f" ({r.get('url') or r.get('link') or r.get('href') or ''})"
|
| 698 |
+
for r in rows[:5]
|
| 699 |
+
)
|
| 700 |
+
evidence.append(f"YOU SEARCHED THE WEB FOR: {query}\nWHAT CAME BACK:\n{block}")
|
| 701 |
+
|
| 702 |
+
# Out of rounds. Rather than a canned apology, make one last call with
|
| 703 |
+
# everything gathered and no way to ask for more, so the operator gets the
|
| 704 |
+
# best answer the evidence supports instead of a shrug.
|
| 705 |
+
final = (await gemini_call_with_checkpoint(
|
| 706 |
+
_compose() + "\n\nYou are out of lookups. Answer them now from what you "
|
| 707 |
+
"have. If it is not enough, say exactly what you could not find.",
|
| 708 |
+
task_type="general", persona=persona, live_state=live,
|
| 709 |
+
can_search=False, can_act=True, can_read=False,
|
| 710 |
+
) or "").strip()
|
| 711 |
+
return final or "I looked, but I couldn't get to a straight answer on that."
|
| 712 |
+
|
| 713 |
+
|
| 714 |
@router.post("/chat")
|
| 715 |
async def chat(request: Request) -> dict[str, Any]:
|
| 716 |
"""Synchronous LLM turn for the mobile app.
|
|
|
|
| 726 |
This runs the prompt synchronously through the token manager's full
|
| 727 |
Gemini→NVIDIA fallback chain and returns the Gemini envelope, so both callers
|
| 728 |
work with no APK change.
|
| 729 |
+
|
| 730 |
+
THE PHONE'S BRAIN IS THIS FUNCTION. Everything the assistant is supposed to
|
| 731 |
+
have — situational judgement about how much to say, refusal to invent system
|
| 732 |
+
state, the ability to look something up — has to be here, because the desktop
|
| 733 |
+
brain in core/brain.py is a different process the handset never reaches. Work
|
| 734 |
+
put only into core/brain.py is, from the phone's point of view, not built.
|
| 735 |
+
|
| 736 |
+
The automation loop is exempt: it wants strict CLICK/INPUT/SCROLL/DONE tokens
|
| 737 |
+
and would be broken by conversational shaping or a lookup detour.
|
| 738 |
"""
|
| 739 |
body = await _json_body(request)
|
| 740 |
text = str(body.get("text") or "").strip()
|
|
|
|
| 744 |
persona = "jarvis"
|
| 745 |
if not text:
|
| 746 |
return _gemini_envelope("Standing by.")
|
| 747 |
+
|
| 748 |
+
structured = bool(context.get("structured")) or str(context.get("mode") or "") == "automation"
|
| 749 |
+
|
| 750 |
try:
|
| 751 |
from backend.services.token_manager import gemini_call_with_checkpoint
|
| 752 |
+
if structured:
|
| 753 |
+
reply = await gemini_call_with_checkpoint(text, task_type="automation", persona=persona)
|
| 754 |
+
return _gemini_envelope(reply or "")
|
| 755 |
+
|
| 756 |
+
live = await _measure_live_state(context)
|
| 757 |
+
raw_history = context.get("history")
|
| 758 |
+
history = raw_history if isinstance(raw_history, list) else []
|
| 759 |
+
reply = await _run_lookup_loop(text, persona, live, history)
|
| 760 |
+
|
| 761 |
+
# Spoken aloud on the handset: strip any markdown that survived the
|
| 762 |
+
# instruction not to write it.
|
| 763 |
+
#
|
| 764 |
+
# An action reply is JSON the app parses and executes, not speech, so it
|
| 765 |
+
# is passed through untouched — running a formatting filter over a
|
| 766 |
+
# payload would corrupt the very thing that carries out the request.
|
| 767 |
+
stripped = reply.lstrip()
|
| 768 |
+
is_action = stripped.startswith("{") and '"actions"' in stripped
|
| 769 |
+
if not is_action:
|
| 770 |
+
try:
|
| 771 |
+
from modules.assistant_identity import to_spoken
|
| 772 |
+
reply = to_spoken(reply)
|
| 773 |
+
except Exception:
|
| 774 |
+
pass
|
| 775 |
return _gemini_envelope(reply or "")
|
| 776 |
except Exception as exc:
|
| 777 |
log.warning("mobile /api/chat LLM call failed: %s", exc)
|
backend/routes/persona_routes.py
CHANGED
|
@@ -52,13 +52,47 @@ async def get_persona():
|
|
| 52 |
|
| 53 |
|
| 54 |
@router.get("/prompts")
|
| 55 |
-
async def get_persona_prompts():
|
| 56 |
-
"""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 57 |
try:
|
| 58 |
-
from modules.assistant_identity import
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 63 |
except Exception as e:
|
| 64 |
raise HTTPException(status_code=500, detail=str(e))
|
|
|
|
| 52 |
|
| 53 |
|
| 54 |
@router.get("/prompts")
|
| 55 |
+
async def get_persona_prompts(full: bool = False):
|
| 56 |
+
"""The persona system prompts.
|
| 57 |
+
|
| 58 |
+
`full=true` returns them COMPLETE, including each persona's own tastes, and
|
| 59 |
+
that is the point of this route now rather than a nicety.
|
| 60 |
+
|
| 61 |
+
assistant_identity.py is meant to be the single source of truth for who the
|
| 62 |
+
assistant is. It is not, in practice: MobileJarvisEngine.kt carries its own
|
| 63 |
+
hardcoded Kotlin copy of the JARVIS characterisation, because an Android app
|
| 64 |
+
cannot import a Python module. Two copies of a personality drift — that is
|
| 65 |
+
exactly how core/brain.py ended up telling JARVIS he was FRIDAY on every
|
| 66 |
+
turn — and the phone's copy is the last one still able to drift.
|
| 67 |
+
|
| 68 |
+
Serving the real text lets the app FETCH the identity and cache it, so there
|
| 69 |
+
is one definition again. Truncating at 500 characters made that impossible,
|
| 70 |
+
which is why the app had a copy in the first place.
|
| 71 |
+
"""
|
| 72 |
try:
|
| 73 |
+
from modules.assistant_identity import (
|
| 74 |
+
JARVIS_PERSONALITY_PROMPT, FRIDAY_PERSONALITY_PROMPT,
|
| 75 |
+
get_identity_persona_prompt, get_mode, set_mode,
|
| 76 |
+
)
|
| 77 |
+
if not full:
|
| 78 |
+
return {
|
| 79 |
+
"jarvis": JARVIS_PERSONALITY_PROMPT[:500] + "...",
|
| 80 |
+
"friday": FRIDAY_PERSONALITY_PROMPT[:500] + "...",
|
| 81 |
+
}
|
| 82 |
+
|
| 83 |
+
# Composed exactly as every other consumer sees it — characterisation
|
| 84 |
+
# plus that persona's own tastes — so the phone gets the same identity
|
| 85 |
+
# the desktop brain and the backend agent get, not a subset.
|
| 86 |
+
original = get_mode()
|
| 87 |
+
out = {}
|
| 88 |
+
try:
|
| 89 |
+
for persona in ("jarvis", "friday"):
|
| 90 |
+
set_mode(persona)
|
| 91 |
+
out[persona] = get_identity_persona_prompt()
|
| 92 |
+
finally:
|
| 93 |
+
set_mode(original)
|
| 94 |
+
|
| 95 |
+
out["active"] = original
|
| 96 |
+
return out
|
| 97 |
except Exception as e:
|
| 98 |
raise HTTPException(status_code=500, detail=str(e))
|
backend/routes/token_task_routes.py
ADDED
|
@@ -0,0 +1,94 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""backend/routes/token_task_routes.py — the token manager, reachable by the phone.
|
| 2 |
+
|
| 3 |
+
WHY THE PHONE MUST NOT HOLD THE KEYS
|
| 4 |
+
------------------------------------
|
| 5 |
+
backend/services/token_manager.py routes each kind of work to its own API key
|
| 6 |
+
domain — OSINT_PROTOCOL_PC, RESEARCH_ENGINE, IMAGE_GENERATION_CLOUD and the
|
| 7 |
+
rest — so that one feature exhausting its quota cannot take the others down
|
| 8 |
+
with it. It also tracks exhausted keys, recovers them on a refresh monitor, and
|
| 9 |
+
checkpoints long tasks so they can resume.
|
| 10 |
+
|
| 11 |
+
The phone has none of that. MobileJarvisEngine.kt calls Gemini directly with a
|
| 12 |
+
single BuildConfig.GEMINI_API_KEY, which means: one key, no domain isolation,
|
| 13 |
+
no rotation, no recovery — and the key is compiled into the APK.
|
| 14 |
+
|
| 15 |
+
The fix is NOT to ship 37 keys to the handset. An APK carrying the master
|
| 16 |
+
credentials is personal-only and can never be distributed; it also puts every
|
| 17 |
+
key on a device that can be lost. Instead the phone asks the backend to do the
|
| 18 |
+
work, and the backend applies the full key strategy on its behalf. The handset
|
| 19 |
+
holds one scoped bearer token; the vault stays where it is.
|
| 20 |
+
|
| 21 |
+
This route is the door. It is deliberately thin: all the intelligence already
|
| 22 |
+
exists in token_manager.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
from typing import Optional
|
| 26 |
+
|
| 27 |
+
from fastapi import APIRouter, HTTPException
|
| 28 |
+
from pydantic import BaseModel
|
| 29 |
+
|
| 30 |
+
router = APIRouter()
|
| 31 |
+
|
| 32 |
+
|
| 33 |
+
class TaskRequest(BaseModel):
|
| 34 |
+
prompt: str
|
| 35 |
+
# Selects the key DOMAIN, not the model. "osint" routes to the OSINT
|
| 36 |
+
# protocol keys, "research" to the research engine keys, and so on, which
|
| 37 |
+
# is the whole point of asking the backend rather than calling Gemini
|
| 38 |
+
# directly from the phone.
|
| 39 |
+
task_type: str = "general"
|
| 40 |
+
persona: str = "jarvis"
|
| 41 |
+
|
| 42 |
+
|
| 43 |
+
class TaskResponse(BaseModel):
|
| 44 |
+
status: str
|
| 45 |
+
result: str
|
| 46 |
+
task_type: str
|
| 47 |
+
persona: str
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
@router.post("/task", response_model=TaskResponse)
|
| 51 |
+
async def run_token_task(req: TaskRequest) -> TaskResponse:
|
| 52 |
+
"""Run a prompt through the token manager, with domain-appropriate keys."""
|
| 53 |
+
prompt = (req.prompt or "").strip()
|
| 54 |
+
if not prompt:
|
| 55 |
+
raise HTTPException(status_code=400, detail="prompt is required")
|
| 56 |
+
|
| 57 |
+
try:
|
| 58 |
+
from backend.services.token_manager import run_task
|
| 59 |
+
result = await run_task(
|
| 60 |
+
prompt=prompt,
|
| 61 |
+
task_type=(req.task_type or "general").strip().lower(),
|
| 62 |
+
persona=(req.persona or "jarvis").strip().lower(),
|
| 63 |
+
)
|
| 64 |
+
return TaskResponse(
|
| 65 |
+
status="ok",
|
| 66 |
+
result=result or "",
|
| 67 |
+
task_type=req.task_type,
|
| 68 |
+
persona=req.persona,
|
| 69 |
+
)
|
| 70 |
+
except Exception as e:
|
| 71 |
+
raise HTTPException(status_code=500, detail=str(e))
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
@router.get("/key_status")
|
| 75 |
+
async def key_status() -> dict:
|
| 76 |
+
"""What the pool looks like right now — counts only, never key material.
|
| 77 |
+
|
| 78 |
+
Useful from the phone precisely because the phone cannot see the vault: it
|
| 79 |
+
is the only way to tell "the feature is broken" from "that domain's quota
|
| 80 |
+
is spent and will recover".
|
| 81 |
+
"""
|
| 82 |
+
try:
|
| 83 |
+
from backend.services import token_manager as tm
|
| 84 |
+
pool = getattr(tm, "GOOGLE_API_KEYS", {}) or {}
|
| 85 |
+
exhausted = getattr(tm, "EXHAUSTED_GOOGLE_KEYS", set()) or set()
|
| 86 |
+
return {
|
| 87 |
+
"status": "ok",
|
| 88 |
+
"domains_loaded": len(pool),
|
| 89 |
+
"domains": sorted(pool.keys()),
|
| 90 |
+
"exhausted_count": len(exhausted),
|
| 91 |
+
"healthy_count": max(0, len(pool) - len(exhausted)),
|
| 92 |
+
}
|
| 93 |
+
except Exception as e:
|
| 94 |
+
raise HTTPException(status_code=500, detail=str(e))
|
backend/services/token_manager.py
CHANGED
|
@@ -1,669 +1,720 @@
|
|
| 1 |
-
# backend/services/token_manager.py
|
| 2 |
-
"""
|
| 3 |
-
OMEGA Token Limit Checkpoint & Resume System
|
| 4 |
-
|
| 5 |
-
When any Gemini 3.5 Flash call hits a ResourceExhausted (429) token limit:
|
| 6 |
-
1. Saves the EXACT task state + all partial output generated so far to SQLite
|
| 7 |
-
2. Launches a background monitor that probes the API every 60s until tokens refresh
|
| 8 |
-
3. On refresh, resumes the task from the EXACT LAST WORD — not from the beginning
|
| 9 |
-
|
| 10 |
-
All Gemini call sites (research, auto_upgrade, captcha, gate check) use
|
| 11 |
-
gemini_call_with_checkpoint() instead of calling the API directly.
|
| 12 |
-
"""
|
| 13 |
-
|
| 14 |
-
import asyncio
|
| 15 |
-
import logging
|
| 16 |
-
import os
|
| 17 |
-
import sqlite3
|
| 18 |
-
import time
|
| 19 |
-
import uuid
|
| 20 |
-
from typing import Optional
|
| 21 |
-
|
| 22 |
-
import google.generativeai as genai
|
| 23 |
-
|
| 24 |
-
from backend.services.usb_vault import KeyDomain, resolve_vault_key
|
| 25 |
-
|
| 26 |
-
def get_runtime_location() -> str:
|
| 27 |
-
return "cloud" if os.environ.get("SPACE_ID") else "pc"
|
| 28 |
-
|
| 29 |
-
# Only the conversational path should be shaped in the assistant's voice.
|
| 30 |
-
# Structured task types (blueprint JSON, code implementation, research digests,
|
| 31 |
-
# captcha/osint payloads) must stay clean — a "You are JARVIS…" preamble there
|
| 32 |
-
# corrupts JSON/code output.
|
| 33 |
-
_PERSONA_TASK_TYPES = {"general", "gaming"}
|
| 34 |
-
|
| 35 |
-
def _persona_system_prompt(persona: str, task_type: str = "general"
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
| 43 |
-
|
| 44 |
-
|
| 45 |
-
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
|
| 54 |
-
|
| 55 |
-
|
| 56 |
-
|
| 57 |
-
|
| 58 |
-
|
| 59 |
-
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
|
| 63 |
-
|
| 64 |
-
|
| 65 |
-
|
| 66 |
-
|
| 67 |
-
|
| 68 |
-
|
| 69 |
-
|
| 70 |
-
|
| 71 |
-
|
| 72 |
-
|
| 73 |
-
|
| 74 |
-
|
| 75 |
-
|
| 76 |
-
|
| 77 |
-
|
| 78 |
-
|
| 79 |
-
|
| 80 |
-
|
| 81 |
-
|
| 82 |
-
|
| 83 |
-
|
| 84 |
-
|
| 85 |
-
|
| 86 |
-
|
| 87 |
-
|
| 88 |
-
|
| 89 |
-
|
| 90 |
-
|
| 91 |
-
|
| 92 |
-
|
| 93 |
-
|
| 94 |
-
|
| 95 |
-
|
| 96 |
-
|
| 97 |
-
|
| 98 |
-
|
| 99 |
-
|
| 100 |
-
|
| 101 |
-
|
| 102 |
-
#
|
| 103 |
-
#
|
| 104 |
-
#
|
| 105 |
-
# gemini-
|
| 106 |
-
# gemini-3
|
| 107 |
-
# gemini-3-flash-
|
| 108 |
-
#
|
| 109 |
-
# The
|
| 110 |
-
#
|
| 111 |
-
|
| 112 |
-
|
| 113 |
-
|
| 114 |
-
|
| 115 |
-
|
| 116 |
-
|
| 117 |
-
|
| 118 |
-
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
| 124 |
-
|
| 125 |
-
|
| 126 |
-
|
| 127 |
-
|
| 128 |
-
|
| 129 |
-
#
|
| 130 |
-
#
|
| 131 |
-
#
|
| 132 |
-
|
| 133 |
-
|
| 134 |
-
|
| 135 |
-
|
| 136 |
-
|
| 137 |
-
|
| 138 |
-
|
| 139 |
-
|
| 140 |
-
|
| 141 |
-
|
| 142 |
-
|
| 143 |
-
|
| 144 |
-
|
| 145 |
-
|
| 146 |
-
|
| 147 |
-
|
| 148 |
-
|
| 149 |
-
|
| 150 |
-
|
| 151 |
-
|
| 152 |
-
|
| 153 |
-
|
| 154 |
-
|
| 155 |
-
|
| 156 |
-
|
| 157 |
-
|
| 158 |
-
|
| 159 |
-
|
| 160 |
-
|
| 161 |
-
|
| 162 |
-
|
| 163 |
-
|
| 164 |
-
|
| 165 |
-
|
| 166 |
-
|
| 167 |
-
|
| 168 |
-
|
| 169 |
-
|
| 170 |
-
|
| 171 |
-
|
| 172 |
-
|
| 173 |
-
|
| 174 |
-
|
| 175 |
-
|
| 176 |
-
|
| 177 |
-
|
| 178 |
-
|
| 179 |
-
|
| 180 |
-
|
| 181 |
-
|
| 182 |
-
|
| 183 |
-
|
| 184 |
-
|
| 185 |
-
|
| 186 |
-
|
| 187 |
-
|
| 188 |
-
|
| 189 |
-
|
| 190 |
-
|
| 191 |
-
|
| 192 |
-
|
| 193 |
-
|
| 194 |
-
|
| 195 |
-
|
| 196 |
-
|
| 197 |
-
|
| 198 |
-
|
| 199 |
-
(
|
| 200 |
-
|
| 201 |
-
|
| 202 |
-
|
| 203 |
-
logging.
|
| 204 |
-
|
| 205 |
-
|
| 206 |
-
|
| 207 |
-
|
| 208 |
-
|
| 209 |
-
|
| 210 |
-
|
| 211 |
-
|
| 212 |
-
|
| 213 |
-
|
| 214 |
-
|
| 215 |
-
|
| 216 |
-
|
| 217 |
-
|
| 218 |
-
|
| 219 |
-
|
| 220 |
-
|
| 221 |
-
|
| 222 |
-
|
| 223 |
-
|
| 224 |
-
|
| 225 |
-
|
| 226 |
-
|
| 227 |
-
|
| 228 |
-
|
| 229 |
-
|
| 230 |
-
|
| 231 |
-
|
| 232 |
-
|
| 233 |
-
|
| 234 |
-
|
| 235 |
-
|
| 236 |
-
|
| 237 |
-
|
| 238 |
-
|
| 239 |
-
|
| 240 |
-
|
| 241 |
-
f"
|
| 242 |
-
|
| 243 |
-
|
| 244 |
-
|
| 245 |
-
|
| 246 |
-
|
| 247 |
-
|
| 248 |
-
|
| 249 |
-
def
|
| 250 |
-
|
| 251 |
-
|
| 252 |
-
|
| 253 |
-
|
| 254 |
-
|
| 255 |
-
|
| 256 |
-
|
| 257 |
-
|
| 258 |
-
|
| 259 |
-
|
| 260 |
-
|
| 261 |
-
|
| 262 |
-
return
|
| 263 |
-
|
| 264 |
-
|
| 265 |
-
|
| 266 |
-
|
| 267 |
-
|
| 268 |
-
|
| 269 |
-
|
| 270 |
-
|
| 271 |
-
|
| 272 |
-
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
| 298 |
-
|
| 299 |
-
|
| 300 |
-
|
| 301 |
-
|
| 302 |
-
|
| 303 |
-
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
| 308 |
-
|
| 309 |
-
|
| 310 |
-
|
| 311 |
-
|
| 312 |
-
|
| 313 |
-
|
| 314 |
-
|
| 315 |
-
|
| 316 |
-
|
| 317 |
-
|
| 318 |
-
|
| 319 |
-
|
| 320 |
-
|
| 321 |
-
|
| 322 |
-
|
| 323 |
-
|
| 324 |
-
|
| 325 |
-
|
| 326 |
-
|
| 327 |
-
|
| 328 |
-
|
| 329 |
-
|
| 330 |
-
|
| 331 |
-
|
| 332 |
-
|
| 333 |
-
|
| 334 |
-
|
| 335 |
-
|
| 336 |
-
|
| 337 |
-
|
| 338 |
-
#
|
| 339 |
-
#
|
| 340 |
-
|
| 341 |
-
|
| 342 |
-
|
| 343 |
-
|
| 344 |
-
|
| 345 |
-
#
|
| 346 |
-
|
| 347 |
-
|
| 348 |
-
|
| 349 |
-
|
| 350 |
-
|
| 351 |
-
|
| 352 |
-
|
| 353 |
-
|
| 354 |
-
|
| 355 |
-
|
| 356 |
-
|
| 357 |
-
|
| 358 |
-
|
| 359 |
-
|
| 360 |
-
|
| 361 |
-
|
| 362 |
-
|
| 363 |
-
|
| 364 |
-
|
| 365 |
-
|
| 366 |
-
|
| 367 |
-
|
| 368 |
-
|
| 369 |
-
|
| 370 |
-
|
| 371 |
-
|
| 372 |
-
|
| 373 |
-
|
| 374 |
-
|
| 375 |
-
|
| 376 |
-
|
| 377 |
-
|
| 378 |
-
|
| 379 |
-
|
| 380 |
-
|
| 381 |
-
|
| 382 |
-
|
| 383 |
-
|
| 384 |
-
|
| 385 |
-
|
| 386 |
-
|
| 387 |
-
|
| 388 |
-
|
| 389 |
-
|
| 390 |
-
|
| 391 |
-
|
| 392 |
-
|
| 393 |
-
|
| 394 |
-
|
| 395 |
-
|
| 396 |
-
|
| 397 |
-
|
| 398 |
-
|
| 399 |
-
|
| 400 |
-
|
| 401 |
-
|
| 402 |
-
|
| 403 |
-
|
| 404 |
-
|
| 405 |
-
|
| 406 |
-
|
| 407 |
-
|
| 408 |
-
|
| 409 |
-
|
| 410 |
-
|
| 411 |
-
|
| 412 |
-
|
| 413 |
-
|
| 414 |
-
|
| 415 |
-
|
| 416 |
-
|
| 417 |
-
|
| 418 |
-
|
| 419 |
-
|
| 420 |
-
|
| 421 |
-
|
| 422 |
-
|
| 423 |
-
|
| 424 |
-
|
| 425 |
-
|
| 426 |
-
|
| 427 |
-
|
| 428 |
-
|
| 429 |
-
|
| 430 |
-
|
| 431 |
-
|
| 432 |
-
|
| 433 |
-
|
| 434 |
-
|
| 435 |
-
|
| 436 |
-
|
| 437 |
-
|
| 438 |
-
|
| 439 |
-
|
| 440 |
-
|
| 441 |
-
|
| 442 |
-
|
| 443 |
-
|
| 444 |
-
|
| 445 |
-
|
| 446 |
-
|
| 447 |
-
|
| 448 |
-
|
| 449 |
-
|
| 450 |
-
|
| 451 |
-
|
| 452 |
-
|
| 453 |
-
|
| 454 |
-
|
| 455 |
-
|
| 456 |
-
|
| 457 |
-
|
| 458 |
-
|
| 459 |
-
|
| 460 |
-
|
| 461 |
-
|
| 462 |
-
|
| 463 |
-
|
| 464 |
-
|
| 465 |
-
|
| 466 |
-
|
| 467 |
-
|
| 468 |
-
|
| 469 |
-
|
| 470 |
-
|
| 471 |
-
|
| 472 |
-
|
| 473 |
-
|
| 474 |
-
|
| 475 |
-
|
| 476 |
-
|
| 477 |
-
|
| 478 |
-
|
| 479 |
-
|
| 480 |
-
|
| 481 |
-
|
| 482 |
-
""
|
| 483 |
-
|
| 484 |
-
|
| 485 |
-
|
| 486 |
-
|
| 487 |
-
|
| 488 |
-
|
| 489 |
-
|
| 490 |
-
|
| 491 |
-
|
| 492 |
-
|
| 493 |
-
|
| 494 |
-
|
| 495 |
-
|
| 496 |
-
|
| 497 |
-
|
| 498 |
-
|
| 499 |
-
|
| 500 |
-
|
| 501 |
-
|
| 502 |
-
|
| 503 |
-
|
| 504 |
-
|
| 505 |
-
|
| 506 |
-
|
| 507 |
-
|
| 508 |
-
|
| 509 |
-
|
| 510 |
-
|
| 511 |
-
|
| 512 |
-
|
| 513 |
-
|
| 514 |
-
|
| 515 |
-
|
| 516 |
-
|
| 517 |
-
|
| 518 |
-
|
| 519 |
-
|
| 520 |
-
|
| 521 |
-
|
| 522 |
-
|
| 523 |
-
|
| 524 |
-
|
| 525 |
-
|
| 526 |
-
|
| 527 |
-
|
| 528 |
-
|
| 529 |
-
|
| 530 |
-
|
| 531 |
-
|
| 532 |
-
|
| 533 |
-
|
| 534 |
-
|
| 535 |
-
|
| 536 |
-
|
| 537 |
-
|
| 538 |
-
|
| 539 |
-
|
| 540 |
-
|
| 541 |
-
|
| 542 |
-
|
| 543 |
-
|
| 544 |
-
|
| 545 |
-
|
| 546 |
-
|
| 547 |
-
|
| 548 |
-
|
| 549 |
-
|
| 550 |
-
|
| 551 |
-
|
| 552 |
-
|
| 553 |
-
|
| 554 |
-
|
| 555 |
-
|
| 556 |
-
|
| 557 |
-
|
| 558 |
-
|
| 559 |
-
|
| 560 |
-
|
| 561 |
-
|
| 562 |
-
|
| 563 |
-
|
| 564 |
-
|
| 565 |
-
|
| 566 |
-
|
| 567 |
-
|
| 568 |
-
|
| 569 |
-
|
| 570 |
-
|
| 571 |
-
|
| 572 |
-
|
| 573 |
-
|
| 574 |
-
|
| 575 |
-
|
| 576 |
-
|
| 577 |
-
|
| 578 |
-
|
| 579 |
-
|
| 580 |
-
|
| 581 |
-
|
| 582 |
-
|
| 583 |
-
|
| 584 |
-
|
| 585 |
-
|
| 586 |
-
|
| 587 |
-
|
| 588 |
-
|
| 589 |
-
|
| 590 |
-
"
|
| 591 |
-
|
| 592 |
-
|
| 593 |
-
|
| 594 |
-
|
| 595 |
-
|
| 596 |
-
|
| 597 |
-
|
| 598 |
-
|
| 599 |
-
|
| 600 |
-
|
| 601 |
-
|
| 602 |
-
|
| 603 |
-
|
| 604 |
-
|
| 605 |
-
|
| 606 |
-
|
| 607 |
-
|
| 608 |
-
|
| 609 |
-
|
| 610 |
-
|
| 611 |
-
|
| 612 |
-
|
| 613 |
-
|
| 614 |
-
|
| 615 |
-
|
| 616 |
-
|
| 617 |
-
|
| 618 |
-
|
| 619 |
-
|
| 620 |
-
|
| 621 |
-
|
| 622 |
-
|
| 623 |
-
|
| 624 |
-
|
| 625 |
-
|
| 626 |
-
|
| 627 |
-
|
| 628 |
-
|
| 629 |
-
|
| 630 |
-
|
| 631 |
-
|
| 632 |
-
|
| 633 |
-
|
| 634 |
-
|
| 635 |
-
|
| 636 |
-
|
| 637 |
-
|
| 638 |
-
|
| 639 |
-
|
| 640 |
-
|
| 641 |
-
|
| 642 |
-
|
| 643 |
-
|
| 644 |
-
|
| 645 |
-
|
| 646 |
-
|
| 647 |
-
|
| 648 |
-
|
| 649 |
-
|
| 650 |
-
|
| 651 |
-
|
| 652 |
-
|
| 653 |
-
|
| 654 |
-
|
| 655 |
-
|
| 656 |
-
|
| 657 |
-
|
| 658 |
-
|
| 659 |
-
|
| 660 |
-
|
| 661 |
-
|
| 662 |
-
|
| 663 |
-
|
| 664 |
-
|
| 665 |
-
|
| 666 |
-
|
| 667 |
-
|
| 668 |
-
|
| 669 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# backend/services/token_manager.py
|
| 2 |
+
"""
|
| 3 |
+
OMEGA Token Limit Checkpoint & Resume System
|
| 4 |
+
|
| 5 |
+
When any Gemini 3.5 Flash call hits a ResourceExhausted (429) token limit:
|
| 6 |
+
1. Saves the EXACT task state + all partial output generated so far to SQLite
|
| 7 |
+
2. Launches a background monitor that probes the API every 60s until tokens refresh
|
| 8 |
+
3. On refresh, resumes the task from the EXACT LAST WORD — not from the beginning
|
| 9 |
+
|
| 10 |
+
All Gemini call sites (research, auto_upgrade, captcha, gate check) use
|
| 11 |
+
gemini_call_with_checkpoint() instead of calling the API directly.
|
| 12 |
+
"""
|
| 13 |
+
|
| 14 |
+
import asyncio
|
| 15 |
+
import logging
|
| 16 |
+
import os
|
| 17 |
+
import sqlite3
|
| 18 |
+
import time
|
| 19 |
+
import uuid
|
| 20 |
+
from typing import Optional
|
| 21 |
+
|
| 22 |
+
import google.generativeai as genai
|
| 23 |
+
|
| 24 |
+
from backend.services.usb_vault import KeyDomain, resolve_vault_key
|
| 25 |
+
|
| 26 |
+
def get_runtime_location() -> str:
|
| 27 |
+
return "cloud" if os.environ.get("SPACE_ID") else "pc"
|
| 28 |
+
|
| 29 |
+
# Only the conversational path should be shaped in the assistant's voice.
|
| 30 |
+
# Structured task types (blueprint JSON, code implementation, research digests,
|
| 31 |
+
# captcha/osint payloads) must stay clean — a "You are JARVIS…" preamble there
|
| 32 |
+
# corrupts JSON/code output.
|
| 33 |
+
_PERSONA_TASK_TYPES = {"general", "gaming"}
|
| 34 |
+
|
| 35 |
+
def _persona_system_prompt(persona: str, task_type: str = "general",
|
| 36 |
+
live_state: Optional[dict] = None,
|
| 37 |
+
can_search: bool = False,
|
| 38 |
+
can_act: bool = False,
|
| 39 |
+
can_read: bool = False) -> str:
|
| 40 |
+
"""The identity prompt for the EXPLICITLY requested persona.
|
| 41 |
+
|
| 42 |
+
The mobile app (and other callers) pass the persona they want per-turn, so
|
| 43 |
+
we select on that string directly rather than the persisted global mode.
|
| 44 |
+
Without this, conversational LLM replies carry no JARVIS/FRIDAY character —
|
| 45 |
+
the identity file that is supposed to be the source of truth was being
|
| 46 |
+
ignored on the primary /api/chat path. Returns "" for non-conversational
|
| 47 |
+
task types so structured output is never polluted.
|
| 48 |
+
|
| 49 |
+
It now composes the FULL identity — characterisation, tastes, behavioural
|
| 50 |
+
judgement and measured live state — rather than the characterisation alone.
|
| 51 |
+
Serving only the characterisation is what produced a JARVIS who opened every
|
| 52 |
+
reply with "at your service", wrote markdown asterisks into speech, and
|
| 53 |
+
announced the PC was connected while it was powered off: he had the
|
| 54 |
+
personality of the character and none of his situational sense, and no facts
|
| 55 |
+
at all. compose_identity() is shared with the desktop brain so the two
|
| 56 |
+
cannot drift apart again.
|
| 57 |
+
"""
|
| 58 |
+
if task_type not in _PERSONA_TASK_TYPES:
|
| 59 |
+
return ""
|
| 60 |
+
try:
|
| 61 |
+
from modules.assistant_identity import compose_identity
|
| 62 |
+
return compose_identity(persona=persona, live_state=live_state,
|
| 63 |
+
can_search=can_search, can_act=can_act, can_read=can_read)
|
| 64 |
+
except Exception:
|
| 65 |
+
# Never lose the identity entirely because state-gathering broke.
|
| 66 |
+
try:
|
| 67 |
+
from modules.assistant_identity import (
|
| 68 |
+
JARVIS_PERSONALITY_PROMPT, FRIDAY_PERSONALITY_PROMPT,
|
| 69 |
+
)
|
| 70 |
+
p = (persona or "jarvis").strip().lower()
|
| 71 |
+
return JARVIS_PERSONALITY_PROMPT if p == "jarvis" else FRIDAY_PERSONALITY_PROMPT
|
| 72 |
+
except Exception:
|
| 73 |
+
return ""
|
| 74 |
+
|
| 75 |
+
# Load all 15 keys into the token manager pool
|
| 76 |
+
GOOGLE_API_KEYS = {}
|
| 77 |
+
try:
|
| 78 |
+
for domain in KeyDomain:
|
| 79 |
+
try:
|
| 80 |
+
key_val = resolve_vault_key(domain)
|
| 81 |
+
if key_val:
|
| 82 |
+
GOOGLE_API_KEYS[domain.value] = key_val
|
| 83 |
+
except Exception:
|
| 84 |
+
continue
|
| 85 |
+
|
| 86 |
+
if get_runtime_location() == "cloud":
|
| 87 |
+
GEMINI_API_KEY = GOOGLE_API_KEYS.get(KeyDomain.CLOUD.value, "")
|
| 88 |
+
else:
|
| 89 |
+
GEMINI_API_KEY = GOOGLE_API_KEYS.get(KeyDomain.CHAT.value, "")
|
| 90 |
+
|
| 91 |
+
if GEMINI_API_KEY:
|
| 92 |
+
genai.configure(api_key=GEMINI_API_KEY)
|
| 93 |
+
else:
|
| 94 |
+
logging.warning("TokenManager: Default API Key is empty.")
|
| 95 |
+
|
| 96 |
+
logging.info(f"TokenManager: Successfully loaded {len(GOOGLE_API_KEYS)} Google/Gemini API keys into the rotation pool.")
|
| 97 |
+
except Exception as e:
|
| 98 |
+
logging.warning(f"TokenManager: API Key initialization failed: {e}")
|
| 99 |
+
GEMINI_API_KEY = ""
|
| 100 |
+
|
| 101 |
+
EXHAUSTED_GOOGLE_KEYS = set()
|
| 102 |
+
# Measured live against a real key, 2026-07-21:
|
| 103 |
+
# gemini-3.5-flash HTTP 503 "currently experiencing high demand" <- was set here
|
| 104 |
+
# gemini-2.0-flash HTTP 429 free-tier request cap
|
| 105 |
+
# gemini-2.5-flash-lite HTTP 404 retired
|
| 106 |
+
# gemini-3-flash-preview OK but 35.5s
|
| 107 |
+
# gemini-3.1-flash-lite OK in 1.1s, correct answer <- chosen
|
| 108 |
+
#
|
| 109 |
+
# The 503 is Google-side capacity, NOT quota consumption: these keys were unused
|
| 110 |
+
# for weeks. Because the primary model always failed, every request fell through
|
| 111 |
+
# to the NVIDIA tier — which is why chat worked at all, and why research paid for
|
| 112 |
+
# a doomed Gemini attempt before doing any real work.
|
| 113 |
+
MODEL = "gemini-3.1-flash-lite"
|
| 114 |
+
|
| 115 |
+
# Google-side model cascade, mirroring the NVIDIA tier cascade in nvidia_vault.
|
| 116 |
+
#
|
| 117 |
+
# A single hardcoded model is fragile in exactly two ways this system has already
|
| 118 |
+
# been bitten by: the model gets retired (gemini-2.5-flash-lite -> 404) or it hits
|
| 119 |
+
# Google-side capacity (gemini-3.5-flash -> 503 "high demand"). Neither means the
|
| 120 |
+
# KEY is bad, and neither should cost a drop to the fallback tier — another
|
| 121 |
+
# Gemini model is usually fine right now.
|
| 122 |
+
#
|
| 123 |
+
# Ordered by what was measured live 2026-07-21 against a real key:
|
| 124 |
+
# gemini-3.1-flash-lite OK, 1.1s, correct answer <- best right now
|
| 125 |
+
# gemini-3.5-flash 503 today, but transient — capacity comes and goes
|
| 126 |
+
# gemini-3-flash-preview OK but 35.5s
|
| 127 |
+
# gemini-2.0-flash 429 free-tier cap today
|
| 128 |
+
# The list is tried in order on any transient failure, and only when every entry
|
| 129 |
+
# fails does the request drop to the NVIDIA tier.
|
| 130 |
+
#
|
| 131 |
+
# Re-verified 2026-08-13 by listing models AND calling each one. Two corrections:
|
| 132 |
+
# gemini-2.0-flash GONE — not in list_models at all, dropped
|
| 133 |
+
# gemini-2.5-flash-lite listed as available, answers 404 on generateContent
|
| 134 |
+
# The second is the trap: list_models is not proof the account may CALL a model,
|
| 135 |
+
# so a dead entry can sit here looking healthy while costing a round trip and a
|
| 136 |
+
# retry on every failover. Verify by calling, never by listing.
|
| 137 |
+
GEMINI_MODEL_CASCADE = [
|
| 138 |
+
MODEL, # gemini-3.1-flash-lite — fastest verified
|
| 139 |
+
"gemini-3.5-flash-lite",
|
| 140 |
+
"gemini-3.5-flash",
|
| 141 |
+
"gemini-2.5-flash",
|
| 142 |
+
"gemini-3-flash-preview", # works, but ~35s
|
| 143 |
+
]
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def _is_transient_error(err_str: str) -> bool:
|
| 147 |
+
"""Transient upstream failure — worth trying another model/tier."""
|
| 148 |
+
return any(t in err_str for t in (
|
| 149 |
+
"resourceexhausted", "429", "quota", "rate limit", "too many requests",
|
| 150 |
+
"503", "500", "high demand", "overloaded", "unavailable",
|
| 151 |
+
"internal error", "deadline", "timeout", "404", "not found",
|
| 152 |
+
"no longer available",
|
| 153 |
+
))
|
| 154 |
+
|
| 155 |
+
# ─────────────────────────────────────────────────────────────────────────────
|
| 156 |
+
# DB Helpers
|
| 157 |
+
# ─────────────────────────────────────────────────────────────────────────────
|
| 158 |
+
|
| 159 |
+
from backend.services.usb_monitor import get_db_path
|
| 160 |
+
|
| 161 |
+
|
| 162 |
+
def _init_checkpoint_table():
|
| 163 |
+
"""Ensure token_checkpoints table exists in memory.db."""
|
| 164 |
+
try:
|
| 165 |
+
with sqlite3.connect(get_db_path()) as conn:
|
| 166 |
+
conn.execute("PRAGMA journal_mode=WAL")
|
| 167 |
+
conn.execute("""
|
| 168 |
+
CREATE TABLE IF NOT EXISTS token_checkpoints (
|
| 169 |
+
id TEXT PRIMARY KEY,
|
| 170 |
+
task_type TEXT NOT NULL,
|
| 171 |
+
original_prompt TEXT NOT NULL,
|
| 172 |
+
partial_result TEXT DEFAULT '',
|
| 173 |
+
last_word TEXT DEFAULT '',
|
| 174 |
+
status TEXT DEFAULT 'pending_resume',
|
| 175 |
+
created_at INTEGER,
|
| 176 |
+
resumed_at INTEGER,
|
| 177 |
+
completed_at INTEGER,
|
| 178 |
+
retry_count INTEGER DEFAULT 0,
|
| 179 |
+
persona TEXT DEFAULT 'jarvis'
|
| 180 |
+
)
|
| 181 |
+
""")
|
| 182 |
+
conn.commit()
|
| 183 |
+
except Exception as e:
|
| 184 |
+
logging.error(f"TokenManager: Failed to init checkpoint table: {e}")
|
| 185 |
+
|
| 186 |
+
|
| 187 |
+
_init_checkpoint_table()
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def _save_checkpoint(task_id: str, task_type: str, prompt: str,
|
| 191 |
+
partial: str, last_word: str, persona: str = "jarvis"):
|
| 192 |
+
try:
|
| 193 |
+
with sqlite3.connect(get_db_path()) as conn:
|
| 194 |
+
conn.execute("PRAGMA journal_mode=WAL")
|
| 195 |
+
conn.execute("""
|
| 196 |
+
INSERT OR REPLACE INTO token_checkpoints
|
| 197 |
+
(id, task_type, original_prompt, partial_result, last_word,
|
| 198 |
+
status, created_at, persona)
|
| 199 |
+
VALUES (?,?,?,?,?,'pending_resume',?,?)
|
| 200 |
+
""", (task_id, task_type, prompt, partial, last_word,
|
| 201 |
+
int(time.time()), persona))
|
| 202 |
+
conn.commit()
|
| 203 |
+
logging.warning(f"TokenManager: Checkpoint saved [{task_type}:{task_id}] last_word='{last_word}'")
|
| 204 |
+
except Exception as e:
|
| 205 |
+
logging.error(f"TokenManager: Failed to save checkpoint: {e}")
|
| 206 |
+
|
| 207 |
+
|
| 208 |
+
def _mark_checkpoint_resumed(task_id: str):
|
| 209 |
+
try:
|
| 210 |
+
with sqlite3.connect(get_db_path()) as conn:
|
| 211 |
+
conn.execute(
|
| 212 |
+
"UPDATE token_checkpoints SET status='resumed', resumed_at=? WHERE id=?",
|
| 213 |
+
(int(time.time()), task_id)
|
| 214 |
+
)
|
| 215 |
+
conn.commit()
|
| 216 |
+
except Exception as e:
|
| 217 |
+
logging.error(f"TokenManager: Failed to mark resumed: {e}")
|
| 218 |
+
|
| 219 |
+
|
| 220 |
+
def _mark_checkpoint_complete(task_id: str):
|
| 221 |
+
try:
|
| 222 |
+
with sqlite3.connect(get_db_path()) as conn:
|
| 223 |
+
conn.execute(
|
| 224 |
+
"UPDATE token_checkpoints SET status='complete', completed_at=? WHERE id=?",
|
| 225 |
+
(int(time.time()), task_id)
|
| 226 |
+
)
|
| 227 |
+
conn.commit()
|
| 228 |
+
except Exception as e:
|
| 229 |
+
logging.error(f"TokenManager: Failed to mark complete: {e}")
|
| 230 |
+
|
| 231 |
+
|
| 232 |
+
def _get_pending_checkpoints() -> list:
|
| 233 |
+
try:
|
| 234 |
+
with sqlite3.connect(get_db_path()) as conn:
|
| 235 |
+
conn.row_factory = sqlite3.Row
|
| 236 |
+
rows = conn.execute(
|
| 237 |
+
"SELECT * FROM token_checkpoints WHERE status='pending_resume' ORDER BY created_at ASC"
|
| 238 |
+
).fetchall()
|
| 239 |
+
return [dict(r) for r in rows]
|
| 240 |
+
except Exception as e:
|
| 241 |
+
logging.error(f"TokenManager: Failed to fetch pending checkpoints: {e}")
|
| 242 |
+
return []
|
| 243 |
+
|
| 244 |
+
|
| 245 |
+
# ─────────────────────────────────────────────────────────────────────────────
|
| 246 |
+
# Core Gemini Call — with auto checkpoint on 429
|
| 247 |
+
# ─────────────────────────────────────────────────────────────────────────────
|
| 248 |
+
|
| 249 |
+
def _extract_last_word(text: str) -> str:
|
| 250 |
+
"""Return the very last word of the partial output for precise resume anchoring."""
|
| 251 |
+
if not text:
|
| 252 |
+
return ""
|
| 253 |
+
words = text.strip().split()
|
| 254 |
+
return words[-1] if words else ""
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def _build_resume_prompt(original_prompt: str, partial_result: str, last_word: str) -> str:
|
| 258 |
+
"""
|
| 259 |
+
Construct a continuation prompt so Gemini resumes from the EXACT last word.
|
| 260 |
+
"""
|
| 261 |
+
if not partial_result:
|
| 262 |
+
return original_prompt
|
| 263 |
+
return (
|
| 264 |
+
f"{original_prompt}\n\n"
|
| 265 |
+
f"---OMEGA RESUME INSTRUCTION---\n"
|
| 266 |
+
f"You were previously generating a response and hit a token limit mid-way.\n"
|
| 267 |
+
f"The partial output generated so far ended with the word: '{last_word}'\n"
|
| 268 |
+
f"Here is the partial output so far:\n\n{partial_result}\n\n"
|
| 269 |
+
f"Continue EXACTLY from where you left off. "
|
| 270 |
+
f"Do NOT repeat any of the partial output above. "
|
| 271 |
+
f"Start your response from the word that comes AFTER '{last_word}'."
|
| 272 |
+
)
|
| 273 |
+
|
| 274 |
+
|
| 275 |
+
def _get_key_for_task(task_type: str) -> str:
|
| 276 |
+
from backend.services.usb_vault import KeyDomain, resolve_vault_key
|
| 277 |
+
location = get_runtime_location()
|
| 278 |
+
|
| 279 |
+
if task_type == "research":
|
| 280 |
+
return resolve_vault_key(KeyDomain.RESEARCH_ENGINE)
|
| 281 |
+
elif task_type == "captcha":
|
| 282 |
+
return resolve_vault_key(KeyDomain.CAPTCHA_SOLVER_CLOUD if location == "cloud" else KeyDomain.CAPTCHA_SOLVER_PC)
|
| 283 |
+
elif task_type == "implement":
|
| 284 |
+
return resolve_vault_key(KeyDomain.AUTO_UPGRADE_CLOUD if location == "cloud" else KeyDomain.AUTO_UPGRADE_PC)
|
| 285 |
+
elif task_type == "osint":
|
| 286 |
+
return resolve_vault_key(KeyDomain.OSINT_PROTOCOL_CLOUD if location == "cloud" else KeyDomain.OSINT_PROTOCOL_PC)
|
| 287 |
+
elif task_type == "image":
|
| 288 |
+
return resolve_vault_key(KeyDomain.IMAGE_GENERATION_CLOUD if location == "cloud" else KeyDomain.IMAGE_GENERATION_PC)
|
| 289 |
+
elif task_type == "gaming":
|
| 290 |
+
return resolve_vault_key(KeyDomain.GAMING_COACH)
|
| 291 |
+
elif task_type == "supervisor":
|
| 292 |
+
return resolve_vault_key(KeyDomain.SUPERVISOR_HEAL_CLOUD if location == "cloud" else KeyDomain.SUPERVISOR_HEAL_PC)
|
| 293 |
+
elif task_type == "cloud_sync":
|
| 294 |
+
return resolve_vault_key(KeyDomain.CLOUD_SYNC_DOMAIN)
|
| 295 |
+
else:
|
| 296 |
+
return resolve_vault_key(KeyDomain.CLOUD if location == "cloud" else KeyDomain.CHAT)
|
| 297 |
+
|
| 298 |
+
def _broadcast_live_key_status(ecosystem: str, model: str, status: str):
|
| 299 |
+
"""Safely fire WS broadcast from synchronous threads"""
|
| 300 |
+
payload = {
|
| 301 |
+
"event": "live_key_status",
|
| 302 |
+
"payload": {
|
| 303 |
+
"active_ecosystem": ecosystem,
|
| 304 |
+
"active_model": model,
|
| 305 |
+
"status": status
|
| 306 |
+
}
|
| 307 |
+
}
|
| 308 |
+
try:
|
| 309 |
+
from backend.ws.agent_ws import ws_manager
|
| 310 |
+
loop = asyncio.get_running_loop()
|
| 311 |
+
loop.create_task(ws_manager.broadcast(payload))
|
| 312 |
+
except RuntimeError:
|
| 313 |
+
pass # No loop running in this thread
|
| 314 |
+
|
| 315 |
+
def _nvidia_fallback_call_sync(prompt: str, task_id: str, task_type: str, partial_so_far: str,
|
| 316 |
+
persona: str, live_state: Optional[dict] = None,
|
| 317 |
+
can_search: bool = False, can_act: bool = False,
|
| 318 |
+
can_read: bool = False) -> str:
|
| 319 |
+
"""Fallback to NVIDIA Vault Models."""
|
| 320 |
+
_broadcast_live_key_status("NVIDIA", "glm-5.1", "fallback_mode")
|
| 321 |
+
|
| 322 |
+
from backend.services.memory_service import GlobalOmniMemory
|
| 323 |
+
omni_context = GlobalOmniMemory.get_global_context_stream()
|
| 324 |
+
|
| 325 |
+
# Construct exact prompt with resume logic and hive mind context
|
| 326 |
+
actual_prompt = _build_resume_prompt(prompt, partial_so_far, _extract_last_word(partial_so_far))
|
| 327 |
+
if not partial_so_far:
|
| 328 |
+
# NVIDIA models take no separate system channel here, so the persona
|
| 329 |
+
# prompt is prepended (same place the hive-mind context goes).
|
| 330 |
+
_sys = _persona_system_prompt(persona, task_type, live_state, can_search, can_act, can_read)
|
| 331 |
+
actual_prompt = (_sys + "\n\n" if _sys else "") + omni_context + actual_prompt
|
| 332 |
+
|
| 333 |
+
# Delegate to the NVIDIA Vault API directly
|
| 334 |
+
from backend.services.nvidia_vault import call_nvidia_model, NvidiaModel
|
| 335 |
+
# GLM is the intended head of the heavy-compute fallback tier. call_nvidia_model
|
| 336 |
+
# already cascades from here through the rest of that tier (MiniMax, DeepSeek,
|
| 337 |
+
# Nemotron-Ultra) with up to 3 key-failovers per model, so a single slow or
|
| 338 |
+
# unavailable model does not strand the request.
|
| 339 |
+
#
|
| 340 |
+
# Only the model ID changed: "z-ai/glm-5.1" was retired upstream and returned
|
| 341 |
+
# HTTP 410 on all 15 keys, which wasted the first attempt of every heavy-tier
|
| 342 |
+
# call. It is now glm-5.2.
|
| 343 |
+
fallback_response = call_nvidia_model(actual_prompt, NvidiaModel.GLM_5_2)
|
| 344 |
+
|
| 345 |
+
# Record what NVIDIA just did to the Hive Mind
|
| 346 |
+
GlobalOmniMemory.record_action("NVIDIA", "glm-5.1", task_type, fallback_response[:100].replace('\n', ' ') + "...")
|
| 347 |
+
|
| 348 |
+
combined = partial_so_far + fallback_response
|
| 349 |
+
_mark_checkpoint_complete(task_id)
|
| 350 |
+
return combined
|
| 351 |
+
|
| 352 |
+
def _gemini_call_sync(prompt: str, task_id: str, task_type: str,
|
| 353 |
+
partial_so_far: str = "", persona: str = "jarvis",
|
| 354 |
+
live_state: Optional[dict] = None,
|
| 355 |
+
can_search: bool = False, can_act: bool = False,
|
| 356 |
+
can_read: bool = False) -> str:
|
| 357 |
+
key = _get_key_for_task(task_type)
|
| 358 |
+
|
| 359 |
+
# 1. If we know it's exhausted, fallback immediately
|
| 360 |
+
if key in EXHAUSTED_GOOGLE_KEYS:
|
| 361 |
+
logging.info(f"TokenManager: Google Key exhausted. Routing {task_type} instantly to NVIDIA Fallback.")
|
| 362 |
+
return _nvidia_fallback_call_sync(prompt, task_id, task_type, partial_so_far,
|
| 363 |
+
persona, live_state, can_search, can_act, can_read)
|
| 364 |
+
|
| 365 |
+
genai.configure(api_key=key)
|
| 366 |
+
_broadcast_live_key_status("GOOGLE", MODEL, "primary_active")
|
| 367 |
+
|
| 368 |
+
from backend.services.memory_service import GlobalOmniMemory
|
| 369 |
+
omni_context = GlobalOmniMemory.get_global_context_stream()
|
| 370 |
+
|
| 371 |
+
# Shape the reply in the active assistant's voice (JARVIS/FRIDAY). Skip on a
|
| 372 |
+
# resume continuation so we don't restate the persona mid-sentence.
|
| 373 |
+
_sys = (_persona_system_prompt(persona, task_type, live_state, can_search, can_act, can_read)
|
| 374 |
+
if not partial_so_far else None)
|
| 375 |
+
actual_prompt = _build_resume_prompt(prompt, partial_so_far, _extract_last_word(partial_so_far))
|
| 376 |
+
if not partial_so_far:
|
| 377 |
+
actual_prompt = omni_context + actual_prompt
|
| 378 |
+
|
| 379 |
+
# Walk the Gemini cascade before considering the NVIDIA tier. A model that is
|
| 380 |
+
# retired or momentarily at capacity must not cost the whole Google side —
|
| 381 |
+
# the next model in the list is usually healthy.
|
| 382 |
+
last_exc = None
|
| 383 |
+
for attempt_model in GEMINI_MODEL_CASCADE:
|
| 384 |
+
try:
|
| 385 |
+
model = (genai.GenerativeModel(attempt_model, system_instruction=_sys)
|
| 386 |
+
if _sys else genai.GenerativeModel(attempt_model))
|
| 387 |
+
response = model.generate_content(actual_prompt)
|
| 388 |
+
new_text = response.text or ""
|
| 389 |
+
|
| 390 |
+
if attempt_model != MODEL:
|
| 391 |
+
logging.warning("TokenManager: primary model unavailable; served by %s", attempt_model)
|
| 392 |
+
_broadcast_live_key_status("GOOGLE", attempt_model, "primary_active")
|
| 393 |
+
|
| 394 |
+
# Record what Google just did to the Hive Mind
|
| 395 |
+
GlobalOmniMemory.record_action("GOOGLE", attempt_model, task_type,
|
| 396 |
+
new_text[:100].replace('\n', ' ') + "...")
|
| 397 |
+
|
| 398 |
+
combined = partial_so_far + new_text
|
| 399 |
+
_mark_checkpoint_complete(task_id)
|
| 400 |
+
return combined
|
| 401 |
+
except Exception as exc:
|
| 402 |
+
last_exc = exc
|
| 403 |
+
if _is_transient_error(str(exc).lower()):
|
| 404 |
+
logging.info("TokenManager: %s unavailable (%s); trying next Gemini model.",
|
| 405 |
+
attempt_model, str(exc)[:70])
|
| 406 |
+
continue
|
| 407 |
+
raise
|
| 408 |
+
|
| 409 |
+
# Every Gemini model failed — fall through to the original handling, which
|
| 410 |
+
# decides between key-exhaustion bookkeeping and the NVIDIA tier.
|
| 411 |
+
try:
|
| 412 |
+
raise last_exc
|
| 413 |
+
except Exception as e:
|
| 414 |
+
err_str = str(e).lower()
|
| 415 |
+
# Any TRANSIENT upstream failure should fall back to NVIDIA, not just a
|
| 416 |
+
# quota error. This previously matched 429/quota only, so a Google-side
|
| 417 |
+
# capacity 503 ("This model is currently experiencing high demand" — which
|
| 418 |
+
# gemini-3.5-flash returns constantly) fell through to `raise` and killed
|
| 419 |
+
# the request outright, never reaching the NVIDIA tier that exists for
|
| 420 |
+
# exactly this situation.
|
| 421 |
+
is_rate_limit = (
|
| 422 |
+
"resourceexhausted" in err_str
|
| 423 |
+
or "429" in err_str
|
| 424 |
+
or "quota" in err_str
|
| 425 |
+
or "rate limit" in err_str
|
| 426 |
+
or "too many requests" in err_str
|
| 427 |
+
# Google-side capacity / availability, not our usage:
|
| 428 |
+
or "503" in err_str
|
| 429 |
+
or "500" in err_str
|
| 430 |
+
or "high demand" in err_str
|
| 431 |
+
or "overloaded" in err_str
|
| 432 |
+
or "unavailable" in err_str
|
| 433 |
+
or "internal error" in err_str
|
| 434 |
+
or "deadline" in err_str
|
| 435 |
+
or "timeout" in err_str
|
| 436 |
+
)
|
| 437 |
+
# A capacity/availability failure is the MODEL's problem, not the key's.
|
| 438 |
+
# Blacklisting the key for a 503 would permanently route a perfectly
|
| 439 |
+
# healthy credential to the fallback tier — so only real quota errors
|
| 440 |
+
# mark a key exhausted. Both still fall back to NVIDIA.
|
| 441 |
+
is_key_exhausted = (
|
| 442 |
+
"resourceexhausted" in err_str
|
| 443 |
+
or "429" in err_str
|
| 444 |
+
or "quota" in err_str
|
| 445 |
+
or "rate limit" in err_str
|
| 446 |
+
or "too many requests" in err_str
|
| 447 |
+
)
|
| 448 |
+
if is_rate_limit:
|
| 449 |
+
logging.warning(
|
| 450 |
+
"TokenManager: transient upstream failure for [%s:%s] (%s). Triggering NVIDIA Fallback.",
|
| 451 |
+
task_type, task_id, "quota" if is_key_exhausted else "capacity/503",
|
| 452 |
+
)
|
| 453 |
+
if not is_key_exhausted:
|
| 454 |
+
# Model-side outage: keep the key healthy, just serve this request
|
| 455 |
+
# from the fallback tier.
|
| 456 |
+
return _nvidia_fallback_call_sync(prompt, task_id, task_type, partial_so_far, persona)
|
| 457 |
+
# 2. Add to exhausted pool
|
| 458 |
+
if key not in EXHAUSTED_GOOGLE_KEYS:
|
| 459 |
+
EXHAUSTED_GOOGLE_KEYS.add(key)
|
| 460 |
+
|
| 461 |
+
# We can't spawn an async task cleanly from a sync thread without a loop,
|
| 462 |
+
# so we will raise a special exception that the async wrapper catches to spawn the pinger.
|
| 463 |
+
raise _Google429Trigger(key=key, prompt=prompt, task_id=task_id, task_type=task_type, partial_so_far=partial_so_far, persona=persona)
|
| 464 |
+
|
| 465 |
+
return _nvidia_fallback_call_sync(prompt, task_id, task_type, partial_so_far, persona)
|
| 466 |
+
raise
|
| 467 |
+
|
| 468 |
+
class _Google429Trigger(Exception):
|
| 469 |
+
def __init__(self, key, prompt, task_id, task_type, partial_so_far, persona):
|
| 470 |
+
self.key = key
|
| 471 |
+
self.prompt = prompt
|
| 472 |
+
self.task_id = task_id
|
| 473 |
+
self.task_type = task_type
|
| 474 |
+
self.partial_so_far = partial_so_far
|
| 475 |
+
self.persona = persona
|
| 476 |
+
|
| 477 |
+
TokenLimitHit = _Google429Trigger
|
| 478 |
+
|
| 479 |
+
async def gemini_call_with_checkpoint(
|
| 480 |
+
prompt: str,
|
| 481 |
+
task_type: str = "general",
|
| 482 |
+
persona: str = "jarvis",
|
| 483 |
+
task_id: Optional[str] = None,
|
| 484 |
+
partial_so_far: str = "",
|
| 485 |
+
live_state: Optional[dict] = None,
|
| 486 |
+
can_search: bool = False,
|
| 487 |
+
can_act: bool = False,
|
| 488 |
+
can_read: bool = False,
|
| 489 |
+
) -> str:
|
| 490 |
+
"""
|
| 491 |
+
Universal Gemini 3.5 Flash call with automatic checkpoint & resume.
|
| 492 |
+
|
| 493 |
+
`live_state` is the set of facts measured for this turn (is the PC actually
|
| 494 |
+
connected, what time is it, what is the device). It reaches the model as part
|
| 495 |
+
of the composed identity. Passing nothing is allowed and honest — the prompt
|
| 496 |
+
then tells him he has no measured facts — but a conversational caller that
|
| 497 |
+
can measure state and does not is how "connected, sir" gets said about a
|
| 498 |
+
machine that is switched off.
|
| 499 |
+
"""
|
| 500 |
+
if task_id is None:
|
| 501 |
+
task_id = str(uuid.uuid4())
|
| 502 |
+
|
| 503 |
+
try:
|
| 504 |
+
return await asyncio.to_thread(
|
| 505 |
+
_gemini_call_sync, prompt, task_id, task_type, partial_so_far, persona,
|
| 506 |
+
live_state, can_search, can_act, can_read
|
| 507 |
+
)
|
| 508 |
+
except _Google429Trigger as e:
|
| 509 |
+
# 1. Start the token refresh monitor loop
|
| 510 |
+
asyncio.create_task(start_token_refresh_monitor())
|
| 511 |
+
# 2. Immediately complete THIS task using NVIDIA fallback
|
| 512 |
+
return await asyncio.to_thread(
|
| 513 |
+
_nvidia_fallback_call_sync, e.prompt, e.task_id, e.task_type, e.partial_so_far,
|
| 514 |
+
e.persona, live_state, can_search, can_act, can_read
|
| 515 |
+
)
|
| 516 |
+
|
| 517 |
+
# For backward compatibility where 'run_task' is imported from token_manager
|
| 518 |
+
async def run_task(prompt: str, task_type: str = "general", persona: str = "jarvis",
|
| 519 |
+
live_state: Optional[dict] = None, can_search: bool = False,
|
| 520 |
+
can_act: bool = False, can_read: bool = False) -> str:
|
| 521 |
+
return await gemini_call_with_checkpoint(prompt, task_type, persona,
|
| 522 |
+
live_state=live_state, can_search=can_search,
|
| 523 |
+
can_act=can_act, can_read=can_read)
|
| 524 |
+
|
| 525 |
+
|
| 526 |
+
# ─────────────────────────────────────────────────────────────────────────────
|
| 527 |
+
# Background Cloud Monitor — polls until tokens refresh then resumes
|
| 528 |
+
# ─────────────────────────────────────────────────────────────────────────────
|
| 529 |
+
|
| 530 |
+
_monitor_running = False
|
| 531 |
+
|
| 532 |
+
async def start_token_refresh_monitor():
|
| 533 |
+
"""
|
| 534 |
+
Background loop that:
|
| 535 |
+
1. Checks for pending_resume checkpoints every 60s
|
| 536 |
+
2. Probes Gemini with a minimal token-cost ping
|
| 537 |
+
3. When the probe succeeds, resumes ALL pending tasks from their exact last word
|
| 538 |
+
4. Broadcasts omega:task_resumed WS event for each resumed task
|
| 539 |
+
"""
|
| 540 |
+
global _monitor_running
|
| 541 |
+
if _monitor_running:
|
| 542 |
+
return
|
| 543 |
+
_monitor_running = True
|
| 544 |
+
logging.info("TokenManager: Token refresh monitor STARTED.")
|
| 545 |
+
|
| 546 |
+
while _monitor_running:
|
| 547 |
+
try:
|
| 548 |
+
pending = _get_pending_checkpoints()
|
| 549 |
+
|
| 550 |
+
# We must probe if there are pending checkpoints OR if we have exhausted keys that need refreshing
|
| 551 |
+
if pending or EXHAUSTED_GOOGLE_KEYS:
|
| 552 |
+
if pending:
|
| 553 |
+
logging.info(f"TokenManager: {len(pending)} checkpoint(s) pending resume. Probing API...")
|
| 554 |
+
else:
|
| 555 |
+
logging.info("TokenManager: No checkpoints, but EXHAUSTED_GOOGLE_KEYS has entries. Probing API...")
|
| 556 |
+
|
| 557 |
+
# Lightweight probe — minimal cost
|
| 558 |
+
# Check each exhausted key independently
|
| 559 |
+
keys_to_probe = list(EXHAUSTED_GOOGLE_KEYS) if EXHAUSTED_GOOGLE_KEYS else [_get_key_for_task("general")]
|
| 560 |
+
recovered_keys = []
|
| 561 |
+
|
| 562 |
+
for key in keys_to_probe:
|
| 563 |
+
try:
|
| 564 |
+
genai.configure(api_key=key)
|
| 565 |
+
model = genai.GenerativeModel(MODEL)
|
| 566 |
+
model.generate_content("ping")
|
| 567 |
+
recovered_keys.append(key)
|
| 568 |
+
except Exception as probe_err:
|
| 569 |
+
err_str = str(probe_err).lower()
|
| 570 |
+
if "resourceexhausted" in err_str or "429" in err_str or "quota" in err_str:
|
| 571 |
+
pass # Still exhausted
|
| 572 |
+
else:
|
| 573 |
+
logging.error(f"TokenManager: Probe error on key: {probe_err}")
|
| 574 |
+
|
| 575 |
+
if not recovered_keys and keys_to_probe:
|
| 576 |
+
logging.warning("TokenManager: API still exhausted. Will retry in 60s.")
|
| 577 |
+
|
| 578 |
+
if recovered_keys:
|
| 579 |
+
logging.info(f"TokenManager: API Probe OK for {len(recovered_keys)} keys. Clearing them from EXHAUSTED_GOOGLE_KEYS.")
|
| 580 |
+
for k in recovered_keys:
|
| 581 |
+
if k in EXHAUSTED_GOOGLE_KEYS:
|
| 582 |
+
EXHAUSTED_GOOGLE_KEYS.remove(k)
|
| 583 |
+
|
| 584 |
+
if pending:
|
| 585 |
+
# Resume all pending checkpoints now that some keys are refreshed
|
| 586 |
+
for checkpoint in pending:
|
| 587 |
+
asyncio.create_task(_resume_checkpoint(checkpoint))
|
| 588 |
+
|
| 589 |
+
except Exception as e:
|
| 590 |
+
logging.error(f"TokenManager: Monitor loop error: {e}")
|
| 591 |
+
|
| 592 |
+
await asyncio.sleep(60)
|
| 593 |
+
|
| 594 |
+
logging.info("TokenManager: Token refresh monitor STOPPED.")
|
| 595 |
+
|
| 596 |
+
|
| 597 |
+
async def _resume_checkpoint(checkpoint: dict):
|
| 598 |
+
"""Resume a single checkpoint from its exact last word."""
|
| 599 |
+
task_id = checkpoint["id"]
|
| 600 |
+
task_type = checkpoint["task_type"]
|
| 601 |
+
original_prompt = checkpoint["original_prompt"]
|
| 602 |
+
partial = checkpoint["partial_result"] or ""
|
| 603 |
+
last_word = checkpoint["last_word"] or ""
|
| 604 |
+
persona = checkpoint.get("persona", "jarvis")
|
| 605 |
+
|
| 606 |
+
logging.info(f"TokenManager: Resuming [{task_type}:{task_id}] from last_word='{last_word}'")
|
| 607 |
+
_mark_checkpoint_resumed(task_id)
|
| 608 |
+
|
| 609 |
+
try:
|
| 610 |
+
# Broadcast WS notification
|
| 611 |
+
from backend.ws.agent_ws import ws_manager
|
| 612 |
+
await ws_manager.broadcast({
|
| 613 |
+
"event": "omega:task_resuming",
|
| 614 |
+
"payload": {
|
| 615 |
+
"task_id": task_id,
|
| 616 |
+
"task_type": task_type,
|
| 617 |
+
"last_word": last_word,
|
| 618 |
+
"message": (
|
| 619 |
+
f"Token limit has refreshed, sir. Resuming {task_type} task from exactly '{last_word}'..."
|
| 620 |
+
if persona == "jarvis" else
|
| 621 |
+
f"We're back boss! Tokens refreshed — resuming {task_type} from exactly '{last_word}'!"
|
| 622 |
+
)
|
| 623 |
+
}
|
| 624 |
+
})
|
| 625 |
+
|
| 626 |
+
# Re-run the Gemini call from exact checkpoint
|
| 627 |
+
result = await gemini_call_with_checkpoint(
|
| 628 |
+
prompt=original_prompt,
|
| 629 |
+
task_type=task_type,
|
| 630 |
+
persona=persona,
|
| 631 |
+
task_id=task_id,
|
| 632 |
+
partial_so_far=partial
|
| 633 |
+
)
|
| 634 |
+
|
| 635 |
+
_mark_checkpoint_complete(task_id)
|
| 636 |
+
|
| 637 |
+
# Route result back to its originating system
|
| 638 |
+
await _dispatch_resumed_result(task_type, task_id, result, persona, checkpoint)
|
| 639 |
+
|
| 640 |
+
await ws_manager.broadcast({
|
| 641 |
+
"event": "omega:task_resumed",
|
| 642 |
+
"payload": {
|
| 643 |
+
"task_id": task_id,
|
| 644 |
+
"task_type": task_type,
|
| 645 |
+
"message": (
|
| 646 |
+
f"{task_type.title()} task completed after token refresh, sir."
|
| 647 |
+
if persona == "jarvis" else
|
| 648 |
+
f"Done boss! {task_type.title()} task finished after the token refresh!"
|
| 649 |
+
)
|
| 650 |
+
}
|
| 651 |
+
})
|
| 652 |
+
|
| 653 |
+
except TokenLimitHit:
|
| 654 |
+
# Still throttled — will retry next monitor cycle
|
| 655 |
+
logging.warning(f"TokenManager: Still throttled on resume for [{task_id}]. Reverting to pending_resume.")
|
| 656 |
+
try:
|
| 657 |
+
with sqlite3.connect(get_db_path()) as conn:
|
| 658 |
+
conn.execute(
|
| 659 |
+
"UPDATE token_checkpoints SET status='pending_resume', retry_count=retry_count+1 WHERE id=?",
|
| 660 |
+
(task_id,)
|
| 661 |
+
)
|
| 662 |
+
conn.commit()
|
| 663 |
+
except Exception as e:
|
| 664 |
+
# No local `import logging` — it would shadow the module import for
|
| 665 |
+
# the whole of _resume_checkpoint() and break the logging calls above.
|
| 666 |
+
logging.getLogger(__name__).error(f"Swallowed exception: {e}")
|
| 667 |
+
except Exception as e:
|
| 668 |
+
logging.error(f"TokenManager: Resume error for [{task_id}]: {e}")
|
| 669 |
+
|
| 670 |
+
|
| 671 |
+
async def _dispatch_resumed_result(task_type: str, task_id: str,
|
| 672 |
+
result: str, persona: str, checkpoint: dict):
|
| 673 |
+
"""Route the completed result back to the correct subsystem."""
|
| 674 |
+
try:
|
| 675 |
+
if task_type == "research":
|
| 676 |
+
# Re-parse the JSON research note and save it
|
| 677 |
+
import json
|
| 678 |
+
from backend.omega.research_engine import ResearchNote, ResearchCategory, _save_research_note
|
| 679 |
+
raw = result.strip().strip("```json").strip("```").strip()
|
| 680 |
+
data = json.loads(raw)
|
| 681 |
+
raw_category = data.get("category", "Research Notes")
|
| 682 |
+
valid_categories = {c.value: c for c in ResearchCategory}
|
| 683 |
+
category = valid_categories.get(raw_category, ResearchCategory.RESEARCH_NOTES)
|
| 684 |
+
note = ResearchNote(
|
| 685 |
+
title=data.get("title", "Resumed Research"),
|
| 686 |
+
summary=data.get("summary", result[:500]),
|
| 687 |
+
importance=data.get("importance", "MEDIUM"),
|
| 688 |
+
category=category,
|
| 689 |
+
source=data.get("source", "Gemini 3.5 Flash (resumed)"),
|
| 690 |
+
recommended_action=data.get("recommended_action", "")
|
| 691 |
+
)
|
| 692 |
+
_save_research_note(note)
|
| 693 |
+
logging.info(f"TokenManager: Research note saved after resume: {note.title}")
|
| 694 |
+
|
| 695 |
+
elif task_type == "implement":
|
| 696 |
+
# Re-parse code blocks and hot-reload them
|
| 697 |
+
from backend.omega.auto_upgrade import parse_code_blocks, write_and_hot_reload, log_upgrade_to_db
|
| 698 |
+
code_blocks = parse_code_blocks(result)
|
| 699 |
+
files_modified = []
|
| 700 |
+
for block in code_blocks:
|
| 701 |
+
await write_and_hot_reload(block.filepath, block.code)
|
| 702 |
+
files_modified.append(block.filepath)
|
| 703 |
+
log_upgrade_to_db(
|
| 704 |
+
checkpoint.get("original_prompt", "Resumed task")[:60],
|
| 705 |
+
files_modified, persona, "success_after_resume"
|
| 706 |
+
)
|
| 707 |
+
logging.info(f"TokenManager: Implementation resumed and hot-reloaded: {files_modified}")
|
| 708 |
+
|
| 709 |
+
else:
|
| 710 |
+
# For gate/captcha/monitor tasks, just log the full result
|
| 711 |
+
logging.info(f"TokenManager: [{task_type}] resumed result stored (length={len(result)})")
|
| 712 |
+
|
| 713 |
+
except Exception as e:
|
| 714 |
+
logging.error(f"TokenManager: dispatch_resumed_result error for [{task_type}]: {e}")
|
| 715 |
+
|
| 716 |
+
|
| 717 |
+
def stop_token_refresh_monitor():
|
| 718 |
+
global _monitor_running
|
| 719 |
+
_monitor_running = False
|
| 720 |
+
logging.info("TokenManager: Token refresh monitor signalled to stop.")
|
backend/services/usb_monitor.py
CHANGED
|
@@ -112,7 +112,9 @@ def get_active_usb_drives():
|
|
| 112 |
"drive_letter": drive_letter
|
| 113 |
})
|
| 114 |
except Exception as e:
|
| 115 |
-
|
|
|
|
|
|
|
| 116 |
logging.error(f"Error enumerating active USB drives: {e}")
|
| 117 |
return drives
|
| 118 |
|
|
@@ -184,7 +186,25 @@ async def start_usb_monitor():
|
|
| 184 |
break
|
| 185 |
if is_pendrive: break
|
| 186 |
except Exception as e:
|
| 187 |
-
import logging
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 188 |
|
| 189 |
if is_pendrive:
|
| 190 |
try:
|
|
|
|
| 112 |
"drive_letter": drive_letter
|
| 113 |
})
|
| 114 |
except Exception as e:
|
| 115 |
+
# Module-scope `logging`, not a local import — the same shadowing trap
|
| 116 |
+
# that broke fetch_pyusb. Harmless here today only because nothing reads
|
| 117 |
+
# `logging` earlier in this function; that is luck, not design.
|
| 118 |
logging.error(f"Error enumerating active USB drives: {e}")
|
| 119 |
return drives
|
| 120 |
|
|
|
|
| 186 |
break
|
| 187 |
if is_pendrive: break
|
| 188 |
except Exception as e:
|
| 189 |
+
# NO local `import logging` here.
|
| 190 |
+
#
|
| 191 |
+
# It was the cause of the error that filled the log
|
| 192 |
+
# every two seconds. A local import binds the name
|
| 193 |
+
# for the WHOLE function, so `logging` became a
|
| 194 |
+
# local of fetch_pyusb — and the outer handler
|
| 195 |
+
# below, which runs when usb.core.find() fails,
|
| 196 |
+
# read it before this line had ever executed:
|
| 197 |
+
#
|
| 198 |
+
# UnboundLocalError: cannot access local variable
|
| 199 |
+
# 'logging' where it is not associated with a value
|
| 200 |
+
#
|
| 201 |
+
# The handler meant to REPORT the PyUSB failure was
|
| 202 |
+
# therefore the thing that crashed, replacing the
|
| 203 |
+
# real reason with a misleading one. `logging` is
|
| 204 |
+
# already imported at module scope; using it is all
|
| 205 |
+
# that was ever needed.
|
| 206 |
+
logging.getLogger(__name__).error(
|
| 207 |
+
"USB interface probe failed: %s", e)
|
| 208 |
|
| 209 |
if is_pendrive:
|
| 210 |
try:
|
backend/tools/web_search_tools.py
CHANGED
|
@@ -217,10 +217,34 @@ async def search_web(query: str, num_results: int = 5) -> dict:
|
|
| 217 |
# worked if it actually returned results; an empty 200 falls through, so a
|
| 218 |
# rate-limited or misconfigured primary degrades instead of silently
|
| 219 |
# returning nothing to the assistant.
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 220 |
for provider, available in ((_brave_search, brave_api_key),
|
| 221 |
(_serpapi_search, serpapi_key),
|
| 222 |
-
# Keyless and
|
| 223 |
-
|
|
|
|
|
|
|
| 224 |
(_wikipedia_search, True)):
|
| 225 |
if not available:
|
| 226 |
continue
|
|
@@ -229,10 +253,7 @@ async def search_web(query: str, num_results: int = 5) -> dict:
|
|
| 229 |
return out
|
| 230 |
logging.warning("web search provider %s failed (%s); trying next",
|
| 231 |
provider.__name__, out.get("error"))
|
| 232 |
-
|
| 233 |
-
# real ranked results, so search/OSINT/threat-modelling still function with
|
| 234 |
-
# no credential configured at all.
|
| 235 |
-
return await asyncio.to_thread(_duck_html_search)
|
| 236 |
|
| 237 |
async def fetch_page(url: str) -> dict:
|
| 238 |
"""Fetch page via Playwright, bypassing CAPTCHAs if encountered"""
|
|
|
|
| 217 |
# worked if it actually returned results; an empty 200 falls through, so a
|
| 218 |
# rate-limited or misconfigured primary degrades instead of silently
|
| 219 |
# returning nothing to the assistant.
|
| 220 |
+
#
|
| 221 |
+
# WIKIPEDIA IS LAST, NOT THIRD.
|
| 222 |
+
#
|
| 223 |
+
# It used to sit above the web scrape, and because it is always "available"
|
| 224 |
+
# it answered nearly every keyless search — which is how OSINT came back
|
| 225 |
+
# confidently wrong. Measured, before this change:
|
| 226 |
+
#
|
| 227 |
+
# "Sundar Pichai" -> 4 Wikipedia articles (Sundar Pichai, Google Gemini,
|
| 228 |
+
# Alphabet Inc., Google). The live web was never asked.
|
| 229 |
+
# "who is Elon Musk" -> "Wealth of Elon Musk", "Errol Musk", "Trump-Musk
|
| 230 |
+
# feud" — an encyclopedia's idea of relevance.
|
| 231 |
+
#
|
| 232 |
+
# An encyclopedia cannot answer the questions this assistant is actually
|
| 233 |
+
# asked: find a specific syllabus PDF, find a person who has a profile but
|
| 234 |
+
# no article, find anything that happened recently. It returned *something*
|
| 235 |
+
# plausible for every query, so the fallback chain never advanced to a real
|
| 236 |
+
# search engine, and the model then wrote an answer out of whatever loosely
|
| 237 |
+
# related articles it was handed. Junk results are worse than no results:
|
| 238 |
+
# empty invites "I couldn't find it", plausible-but-wrong invites invention.
|
| 239 |
+
#
|
| 240 |
+
# Below the scrape it still does its real job — the thing that keeps research
|
| 241 |
+
# alive when DuckDuckGo throws its anti-automation challenge.
|
| 242 |
for provider, available in ((_brave_search, brave_api_key),
|
| 243 |
(_serpapi_search, serpapi_key),
|
| 244 |
+
# Keyless, and actual ranked web results.
|
| 245 |
+
(_duck_html_search, True),
|
| 246 |
+
# Keyless and unthrottled: the safety net for when
|
| 247 |
+
# the scrape above is challenged, not the default.
|
| 248 |
(_wikipedia_search, True)):
|
| 249 |
if not available:
|
| 250 |
continue
|
|
|
|
| 253 |
return out
|
| 254 |
logging.warning("web search provider %s failed (%s); trying next",
|
| 255 |
provider.__name__, out.get("error"))
|
| 256 |
+
return {"error": "no search provider returned results"}
|
|
|
|
|
|
|
|
|
|
| 257 |
|
| 258 |
async def fetch_page(url: str) -> dict:
|
| 259 |
"""Fetch page via Playwright, bypassing CAPTCHAs if encountered"""
|
backend/voice/engines/kokoro_engine.py
CHANGED
|
The diff for this file is too large to render.
See raw diff
|
|
|
backend/voice/jarvis_live.py
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""The JARVIS voice for everything JARVIS actually says.
|
| 2 |
+
|
| 3 |
+
`backend/voice/tts.py` already routes the *backend's* JARVIS speech through the
|
| 4 |
+
Kokoro+RVC pipeline. But the desktop assistant does not speak through that
|
| 5 |
+
path at all — tray.py, modules/executor.py, modules/hud.py, the automation
|
| 6 |
+
triggers, the Gmail connector and the tour all call `core.voice.speak()`, which
|
| 7 |
+
synthesises with Edge-TTS. So the tuned voice was reachable from the API and
|
| 8 |
+
from the phone, and from nowhere the user actually hears day to day.
|
| 9 |
+
|
| 10 |
+
This module is the bridge. It renders a line with the real pipeline and hands
|
| 11 |
+
back a WAV path that `core.voice`'s existing player can use unchanged.
|
| 12 |
+
|
| 13 |
+
Three things make that practical rather than merely possible:
|
| 14 |
+
|
| 15 |
+
* A DISK CACHE. Assistants repeat themselves — "Done.", "Yes, sir.",
|
| 16 |
+
"Standing by." A repeated line costs a file read instead of seconds of
|
| 17 |
+
inference, which is what makes short acknowledgements usable at all.
|
| 18 |
+
|
| 19 |
+
* A WARM-UP. The first render pays ~35 s of model loading. Left alone, the
|
| 20 |
+
first thing JARVIS says after boot arrives half a minute late. `warm_up()`
|
| 21 |
+
moves that cost to startup, where nothing is waiting on it.
|
| 22 |
+
|
| 23 |
+
* A FALLBACK THAT NEVER MUTES HIM. If torch, RVC or the model files are
|
| 24 |
+
missing — a stripped exe build, a fresh machine — this returns None and the
|
| 25 |
+
caller keeps its Edge-TTS path. A degraded voice is recoverable; silence
|
| 26 |
+
is not.
|
| 27 |
+
"""
|
| 28 |
+
|
| 29 |
+
from __future__ import annotations
|
| 30 |
+
|
| 31 |
+
import hashlib
|
| 32 |
+
import logging
|
| 33 |
+
import os
|
| 34 |
+
import threading
|
| 35 |
+
import time
|
| 36 |
+
|
| 37 |
+
logger = logging.getLogger(__name__)
|
| 38 |
+
|
| 39 |
+
# ── Configuration ────────────────────────────────────────────────────────────
|
| 40 |
+
|
| 41 |
+
# The music bed is OFF for live speech, and that is deliberate.
|
| 42 |
+
#
|
| 43 |
+
# The bed exists to match the reference recording, which is a scored piece of
|
| 44 |
+
# film audio. Under a 40-second demo it is the thing that made the clip sound
|
| 45 |
+
# like the real JARVIS. Under every "Done." and every alert, all day, a looping
|
| 46 |
+
# 40-second cue would be exhausting — and each utterance would restart the loop
|
| 47 |
+
# from the same point, so the same few bars would play over and over.
|
| 48 |
+
#
|
| 49 |
+
# Set to 5.0 to get the demo's bed back; nothing else needs to change.
|
| 50 |
+
LIVE_AMBIENCE_LEVEL = 0.0
|
| 51 |
+
|
| 52 |
+
# Bump when the voice itself changes, so old cache entries are not reused for
|
| 53 |
+
# a voice that no longer exists. This is a cache key component, not a version
|
| 54 |
+
# number anyone reads.
|
| 55 |
+
VOICE_REVISION = "2026-08-11-a"
|
| 56 |
+
|
| 57 |
+
# Lines longer than this are not worth caching — they are one-off sentences
|
| 58 |
+
# that will never repeat, and caching them only fills the disk.
|
| 59 |
+
CACHE_MAX_CHARS = 240
|
| 60 |
+
CACHE_MAX_FILES = 400
|
| 61 |
+
|
| 62 |
+
_lock = threading.Lock()
|
| 63 |
+
_warm_thread: threading.Thread | None = None
|
| 64 |
+
_available: bool | None = None # None = not yet determined
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
def _cache_dir() -> str:
|
| 68 |
+
base = os.environ.get("JARVIS_APP_DATA_DIR") or os.environ.get("LOCALAPPDATA") \
|
| 69 |
+
or os.path.expanduser("~")
|
| 70 |
+
d = os.path.join(base, "jarvis_voice_cache")
|
| 71 |
+
os.makedirs(d, exist_ok=True)
|
| 72 |
+
return d
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _key(text: str, profile: str) -> str:
|
| 76 |
+
raw = f"{VOICE_REVISION}|{profile}|{LIVE_AMBIENCE_LEVEL}|{text}"
|
| 77 |
+
return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:32]
|
| 78 |
+
|
| 79 |
+
|
| 80 |
+
def is_available() -> bool:
|
| 81 |
+
"""Can the real pipeline run here?
|
| 82 |
+
|
| 83 |
+
Answered by importing, not by looking for files: the failure modes that
|
| 84 |
+
matter (no torch, no CUDA runtime, a broken faiss/numpy pairing) are all
|
| 85 |
+
import-time, and every one of them has silently degraded this pipeline to
|
| 86 |
+
plain Kokoro before.
|
| 87 |
+
"""
|
| 88 |
+
global _available
|
| 89 |
+
if _available is not None:
|
| 90 |
+
return _available
|
| 91 |
+
try:
|
| 92 |
+
import backend.voice.engines.kokoro_engine as ke # noqa: F401
|
| 93 |
+
_available = True
|
| 94 |
+
except Exception as exc:
|
| 95 |
+
logger.warning("[JARVIS voice] pipeline unavailable (%s); "
|
| 96 |
+
"falling back to Edge-TTS.", exc)
|
| 97 |
+
_available = False
|
| 98 |
+
return _available
|
| 99 |
+
|
| 100 |
+
|
| 101 |
+
def warm_up() -> None:
|
| 102 |
+
"""Load the models in the background so the first line is not the slow one.
|
| 103 |
+
|
| 104 |
+
Idempotent and non-blocking. Safe to call from startup paths that must not
|
| 105 |
+
wait for anything.
|
| 106 |
+
"""
|
| 107 |
+
global _warm_thread
|
| 108 |
+
with _lock:
|
| 109 |
+
if _warm_thread is not None:
|
| 110 |
+
return
|
| 111 |
+
|
| 112 |
+
def _run():
|
| 113 |
+
try:
|
| 114 |
+
t0 = time.time()
|
| 115 |
+
import backend.voice.engines.kokoro_engine as ke
|
| 116 |
+
ke._get_kokoro()
|
| 117 |
+
ke._get_rvc()
|
| 118 |
+
logger.info("[JARVIS voice] models warm in %.1fs", time.time() - t0)
|
| 119 |
+
except Exception as exc:
|
| 120 |
+
logger.warning("[JARVIS voice] warm-up failed: %s", exc)
|
| 121 |
+
|
| 122 |
+
_warm_thread = threading.Thread(target=_run, daemon=True,
|
| 123 |
+
name="jarvis-voice-warmup")
|
| 124 |
+
_warm_thread.start()
|
| 125 |
+
|
| 126 |
+
|
| 127 |
+
def _prune_cache(d: str) -> None:
|
| 128 |
+
"""Keep the cache bounded, oldest-accessed first."""
|
| 129 |
+
try:
|
| 130 |
+
files = [os.path.join(d, f) for f in os.listdir(d) if f.endswith(".wav")]
|
| 131 |
+
if len(files) <= CACHE_MAX_FILES:
|
| 132 |
+
return
|
| 133 |
+
files.sort(key=lambda p: os.path.getatime(p))
|
| 134 |
+
for p in files[:len(files) - CACHE_MAX_FILES]:
|
| 135 |
+
try:
|
| 136 |
+
os.remove(p)
|
| 137 |
+
except OSError:
|
| 138 |
+
pass
|
| 139 |
+
except Exception:
|
| 140 |
+
pass
|
| 141 |
+
|
| 142 |
+
|
| 143 |
+
def render_to_file(text: str, profile: str | None = None) -> str | None:
|
| 144 |
+
"""Render `text` in the JARVIS voice; return a WAV path, or None.
|
| 145 |
+
|
| 146 |
+
None means "use your existing fallback" — it is never an error the caller
|
| 147 |
+
has to handle beyond that.
|
| 148 |
+
"""
|
| 149 |
+
text = (text or "").strip()
|
| 150 |
+
if not text or not is_available():
|
| 151 |
+
return None
|
| 152 |
+
|
| 153 |
+
try:
|
| 154 |
+
import backend.voice.engines.kokoro_engine as ke
|
| 155 |
+
except Exception:
|
| 156 |
+
return None
|
| 157 |
+
|
| 158 |
+
name = profile or getattr(ke, "ACTIVE_PROFILE", "m4a")
|
| 159 |
+
cacheable = len(text) <= CACHE_MAX_CHARS
|
| 160 |
+
d = _cache_dir()
|
| 161 |
+
path = os.path.join(d, _key(text, name) + ".wav")
|
| 162 |
+
|
| 163 |
+
if cacheable and os.path.exists(path) and os.path.getsize(path) > 1024:
|
| 164 |
+
try:
|
| 165 |
+
os.utime(path, None) # keep it fresh for the pruner
|
| 166 |
+
except OSError:
|
| 167 |
+
pass
|
| 168 |
+
return path
|
| 169 |
+
|
| 170 |
+
try:
|
| 171 |
+
import asyncio
|
| 172 |
+
|
| 173 |
+
# The bed level is a module global on the engine, and the engine has no
|
| 174 |
+
# per-call override for it. Set it around the call and put it back, so
|
| 175 |
+
# a caller that renders WITH a bed (the demo scripts) is unaffected.
|
| 176 |
+
prev = getattr(ke, "AMBIENCE_LEVEL", 0.0)
|
| 177 |
+
ke.AMBIENCE_LEVEL = LIVE_AMBIENCE_LEVEL
|
| 178 |
+
try:
|
| 179 |
+
t0 = time.time()
|
| 180 |
+
data = asyncio.run(ke.synthesize_kokoro(text, profile=name))
|
| 181 |
+
finally:
|
| 182 |
+
ke.AMBIENCE_LEVEL = prev
|
| 183 |
+
|
| 184 |
+
if not data or len(data) < 1024:
|
| 185 |
+
logger.warning("[JARVIS voice] empty render for %r", text[:40])
|
| 186 |
+
return None
|
| 187 |
+
|
| 188 |
+
# Write to a temp name and rename, so a cache hit can never land on a
|
| 189 |
+
# half-written file if two threads race on the same line.
|
| 190 |
+
tmp = path + f".{os.getpid()}.{threading.get_ident()}.part"
|
| 191 |
+
with open(tmp, "wb") as fh:
|
| 192 |
+
fh.write(data)
|
| 193 |
+
os.replace(tmp, path)
|
| 194 |
+
|
| 195 |
+
logger.info("[JARVIS voice] rendered %d chars in %.2fs", len(text),
|
| 196 |
+
time.time() - t0)
|
| 197 |
+
if cacheable:
|
| 198 |
+
_prune_cache(d)
|
| 199 |
+
return path
|
| 200 |
+
except Exception as exc:
|
| 201 |
+
logger.warning("[JARVIS voice] render failed (%s); falling back.", exc)
|
| 202 |
+
return None
|
backend/voice/tts.py
CHANGED
|
@@ -41,6 +41,18 @@ class TTSPipeline:
|
|
| 41 |
"""
|
| 42 |
import logging
|
| 43 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 44 |
# ── FRIDAY path — completely untouched, existing logic, zero changes ──
|
| 45 |
if personality != "jarvis":
|
| 46 |
return await self._synthesize_friday(text, personality, language)
|
|
|
|
| 41 |
"""
|
| 42 |
import logging
|
| 43 |
|
| 44 |
+
# Numbers become words HERE, before the engine split.
|
| 45 |
+
#
|
| 46 |
+
# This lived inside kokoro_engine, which meant only the JARVIS English
|
| 47 |
+
# path ever got it — FRIDAY, the Edge-TTS fallback and every non-English
|
| 48 |
+
# language still read "15,444" as "fifteen" then "zero zero zero",
|
| 49 |
+
# because the comma splits the token and the rest is voiced digit by
|
| 50 |
+
# digit. This is the one place every engine passes through, so it is the
|
| 51 |
+
# only place the fix belongs. Applying it twice is harmless: after the
|
| 52 |
+
# first pass there are no digits left to match.
|
| 53 |
+
from backend.voice.engines.kokoro_engine import speak_numbers
|
| 54 |
+
text = speak_numbers(text)
|
| 55 |
+
|
| 56 |
# ── FRIDAY path — completely untouched, existing logic, zero changes ──
|
| 57 |
if personality != "jarvis":
|
| 58 |
return await self._synthesize_friday(text, personality, language)
|
config.py
CHANGED
|
@@ -433,8 +433,78 @@ ANALYTICS_DIR = os.path.join(COPIES_DIR, "analytics")
|
|
| 433 |
REPAIR_DIR = os.path.join(COPIES_DIR, "repair")
|
| 434 |
|
| 435 |
# API
|
| 436 |
-
|
| 437 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 438 |
|
| 439 |
# Audio / Voice
|
| 440 |
# FRIDAY OMEGA - Young, Natural Human Voice (20-25 female)
|
|
|
|
| 433 |
REPAIR_DIR = os.path.join(COPIES_DIR, "repair")
|
| 434 |
|
| 435 |
# API
|
| 436 |
+
#
|
| 437 |
+
# This was the literal `GEMINI_API_KEY = ""` with a note to paste a key in, and
|
| 438 |
+
# nothing ever read .env — which has held a valid key the whole time. The cost
|
| 439 |
+
# was not a warning in a log. core/brain.py imports this name, _ask_gemini
|
| 440 |
+
# raised on every single call, and core/brain_router fell through to the
|
| 441 |
+
# provider named "local" — which is NOT a local model, just canned strings from
|
| 442 |
+
# _local_fallback(). So every "reply" from JARVIS or FRIDAY on the desktop was
|
| 443 |
+
# stock filler, including the "Got it: <x>. Tell me the outcome you want." that
|
| 444 |
+
# core/commands.py has a comment complaining about.
|
| 445 |
+
#
|
| 446 |
+
# Environment first (a Space secret or a shell export must win), then .env, then
|
| 447 |
+
# the literal, so a real deployment can override without editing source.
|
| 448 |
+
def _read_env_file(name: str) -> str:
|
| 449 |
+
try:
|
| 450 |
+
env_path = Path(__file__).resolve().parent / ".env"
|
| 451 |
+
if not env_path.exists():
|
| 452 |
+
return ""
|
| 453 |
+
for raw in env_path.read_text(encoding="utf-8", errors="replace").splitlines():
|
| 454 |
+
line = raw.strip()
|
| 455 |
+
if not line or line.startswith("#") or "=" not in line:
|
| 456 |
+
continue
|
| 457 |
+
key, _, value = line.partition("=")
|
| 458 |
+
if key.strip() == name:
|
| 459 |
+
return value.strip().strip('"').strip("'")
|
| 460 |
+
except Exception:
|
| 461 |
+
pass
|
| 462 |
+
return ""
|
| 463 |
+
|
| 464 |
+
|
| 465 |
+
GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY") or _read_env_file("GEMINI_API_KEY") or ""
|
| 466 |
+
# Two failure modes have to be avoided here, and they pull in opposite
|
| 467 |
+
# directions.
|
| 468 |
+
#
|
| 469 |
+
# The original value was "gemini-2.5-flash-preview-04-17" — a dated preview that
|
| 470 |
+
# has since been RETIRED and answered every call with HTTP 404. Pinning a
|
| 471 |
+
# preview is a time bomb.
|
| 472 |
+
#
|
| 473 |
+
# The obvious fix, "gemini-flash-latest", is worse for this system: it resolves
|
| 474 |
+
# to whatever is newest, and the newest model carries the TIGHTEST free-tier
|
| 475 |
+
# quota. Measured: gemini-3.6-flash allows 20 requests PER DAY. An always-on
|
| 476 |
+
# assistant exhausts that before lunch and then runs on canned fallback strings
|
| 477 |
+
# for the rest of the day, silently, because brain_router swallows the 429.
|
| 478 |
+
#
|
| 479 |
+
# So: a stable, non-preview model with a workable free allowance, plus a
|
| 480 |
+
# fallback chain for when its quota is gone. Capability matters less than being
|
| 481 |
+
# answered at all — a slightly older model that replies beats a newer one that
|
| 482 |
+
# 429s.
|
| 483 |
+
GEMINI_MODEL = (os.environ.get("GEMINI_MODEL")
|
| 484 |
+
or _read_env_file("GEMINI_MODEL")
|
| 485 |
+
or "gemini-2.5-flash")
|
| 486 |
+
|
| 487 |
+
# Tried in order when the primary is rate-limited. Lite models have materially
|
| 488 |
+
# larger free quotas and are perfectly adequate for planning and judgement.
|
| 489 |
+
# Verified against list_models AND a real generateContent call on 2026-08-13.
|
| 490 |
+
# Three of the previous five were unusable: gemini-2.0-flash and
|
| 491 |
+
# gemini-2.0-flash-lite are gone entirely, and gemini-2.5-flash-lite is listed as
|
| 492 |
+
# available yet answers 404 "no longer available to new users" when called. A
|
| 493 |
+
# dead entry is not free — the cascade spends an attempt, a round trip and a
|
| 494 |
+
# retry on each one before reaching a model that works, which is why failover
|
| 495 |
+
# felt like an outage rather than a fallback.
|
| 496 |
+
#
|
| 497 |
+
# Verify the same way when this changes: listing a model does not prove the
|
| 498 |
+
# account may call it.
|
| 499 |
+
GEMINI_MODEL_FALLBACKS = [
|
| 500 |
+
"gemini-2.5-flash", # primary; 5 req/min on free tier
|
| 501 |
+
"gemini-3.5-flash-lite",
|
| 502 |
+
"gemini-3.1-flash-lite",
|
| 503 |
+
"gemini-3.5-flash",
|
| 504 |
+
# Deliberately last: gemini-3.6-flash is capped at ~20 requests per DAY on
|
| 505 |
+
# the free tier, so promoting it silently kills the brain by lunchtime.
|
| 506 |
+
"gemini-3.6-flash",
|
| 507 |
+
]
|
| 508 |
|
| 509 |
# Audio / Voice
|
| 510 |
# FRIDAY OMEGA - Young, Natural Human Voice (20-25 female)
|
modules/alert_manager.py
ADDED
|
@@ -0,0 +1,307 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Alerts that behave like a person raising something, not a loop shouting.
|
| 2 |
+
|
| 3 |
+
The watch loops in OMEGA announce a condition every time they observe it. A
|
| 4 |
+
condition that persists — a disk filling, a device still disconnected — is
|
| 5 |
+
therefore announced again on every pass, forever, at whatever interval that
|
| 6 |
+
loop happens to run. The existing `cooldown_s` in core.voice only spaces the
|
| 7 |
+
repeats out; it never stops them, and it is per-process, so two components
|
| 8 |
+
watching the same thing each get their own cooldown.
|
| 9 |
+
|
| 10 |
+
What is wanted instead:
|
| 11 |
+
|
| 12 |
+
* say it once;
|
| 13 |
+
* if it is not acknowledged, raise it again after a short while, because
|
| 14 |
+
something genuinely unattended should not be dropped silently;
|
| 15 |
+
* back off each time rather than nagging at a fixed rate;
|
| 16 |
+
* stop entirely once acknowledged, or once told to wait, or once muted;
|
| 17 |
+
* accept "not now", "tell me in 30 minutes" and "don't alert me about this
|
| 18 |
+
for two hours" as the way to control all of that.
|
| 19 |
+
|
| 20 |
+
State is on disk, keyed by TOPIC rather than by message text, so a rephrased
|
| 21 |
+
message about the same subject is still the same alert — "disk 91% full" and
|
| 22 |
+
"disk 93% full" must not read as two different things to nag about.
|
| 23 |
+
"""
|
| 24 |
+
|
| 25 |
+
from __future__ import annotations
|
| 26 |
+
|
| 27 |
+
import json
|
| 28 |
+
import os
|
| 29 |
+
import re
|
| 30 |
+
import threading
|
| 31 |
+
import time
|
| 32 |
+
from typing import Optional
|
| 33 |
+
|
| 34 |
+
_STATE_DIR = os.environ.get("JARVIS_APP_DATA_DIR") or os.path.join(
|
| 35 |
+
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".friday_data")
|
| 36 |
+
_STATE_PATH = os.path.join(_STATE_DIR, "alert_state.json")
|
| 37 |
+
|
| 38 |
+
_lock = threading.RLock()
|
| 39 |
+
|
| 40 |
+
# Escalation ladder for an unacknowledged alert, in seconds.
|
| 41 |
+
#
|
| 42 |
+
# The first repeat is the 5 minutes asked for. After that it doubles and then
|
| 43 |
+
# stops: three unanswered mentions is the point at which repeating again is
|
| 44 |
+
# nagging rather than diligence. A genuinely critical condition should be
|
| 45 |
+
# raised with importance="critical", which uses the longer ladder below and
|
| 46 |
+
# does not give up.
|
| 47 |
+
REPEAT_LADDER = (300, 900, 2700)
|
| 48 |
+
CRITICAL_LADDER = (120, 300, 900, 1800, 3600)
|
| 49 |
+
|
| 50 |
+
# How long an alert stays "open" waiting to be acknowledged before it is
|
| 51 |
+
# considered stale and dropped.
|
| 52 |
+
MAX_ALERT_AGE = 24 * 3600
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
def _load() -> dict:
|
| 56 |
+
try:
|
| 57 |
+
with open(_STATE_PATH, "r", encoding="utf-8") as fh:
|
| 58 |
+
data = json.load(fh)
|
| 59 |
+
return data if isinstance(data, dict) else {}
|
| 60 |
+
except Exception:
|
| 61 |
+
return {}
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _save(data: dict) -> None:
|
| 65 |
+
try:
|
| 66 |
+
os.makedirs(_STATE_DIR, exist_ok=True)
|
| 67 |
+
tmp = _STATE_PATH + ".tmp"
|
| 68 |
+
with open(tmp, "w", encoding="utf-8") as fh:
|
| 69 |
+
json.dump(data, fh, indent=2)
|
| 70 |
+
os.replace(tmp, _STATE_PATH)
|
| 71 |
+
except Exception:
|
| 72 |
+
pass
|
| 73 |
+
|
| 74 |
+
|
| 75 |
+
def _topics(data: dict) -> dict:
|
| 76 |
+
return data.setdefault("topics", {})
|
| 77 |
+
|
| 78 |
+
|
| 79 |
+
# ── Total silence ───────────────────────────────────────────────────────────
|
| 80 |
+
#
|
| 81 |
+
# Distinct from mute(topic, seconds) above, which is per-subject and expires.
|
| 82 |
+
# "Mute" said out loud does not mean "quieten one topic for a while" — it means
|
| 83 |
+
# say nothing at all until told otherwise. There was no such state anywhere in
|
| 84 |
+
# the system: saying "mute" hit core.commands' instant_commands table and muted
|
| 85 |
+
# the WINDOWS MIXER, so JARVIS carried on speaking into a silenced output and
|
| 86 |
+
# came back at full volume the moment anything else unmuted the device.
|
| 87 |
+
#
|
| 88 |
+
# This latch has no timeout by design. An indefinite silence that expires on
|
| 89 |
+
# its own is not the thing that was asked for.
|
| 90 |
+
|
| 91 |
+
def silence_all(reason: str = "") -> None:
|
| 92 |
+
with _lock:
|
| 93 |
+
data = _load()
|
| 94 |
+
data["silenced"] = {"since": time.time(), "reason": reason or "asked"}
|
| 95 |
+
_save(data)
|
| 96 |
+
|
| 97 |
+
|
| 98 |
+
def unsilence_all() -> None:
|
| 99 |
+
with _lock:
|
| 100 |
+
data = _load()
|
| 101 |
+
data.pop("silenced", None)
|
| 102 |
+
_save(data)
|
| 103 |
+
|
| 104 |
+
|
| 105 |
+
def is_silenced() -> bool:
|
| 106 |
+
try:
|
| 107 |
+
return bool(_load().get("silenced"))
|
| 108 |
+
except Exception:
|
| 109 |
+
return False
|
| 110 |
+
|
| 111 |
+
|
| 112 |
+
def silence_info() -> Optional[dict]:
|
| 113 |
+
try:
|
| 114 |
+
got = _load().get("silenced")
|
| 115 |
+
return got if isinstance(got, dict) else None
|
| 116 |
+
except Exception:
|
| 117 |
+
return None
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def should_speak(topic: str, importance: str = "normal") -> bool:
|
| 121 |
+
"""May this topic be spoken right now?
|
| 122 |
+
|
| 123 |
+
Called by the raising code BEFORE it speaks. Returns False when the topic
|
| 124 |
+
is muted, snoozed, already acknowledged, or simply not due for its next
|
| 125 |
+
repeat yet — which is what turns a per-pass watch loop into an alert that
|
| 126 |
+
is mentioned a sensible number of times.
|
| 127 |
+
"""
|
| 128 |
+
now = time.time()
|
| 129 |
+
with _lock:
|
| 130 |
+
data = _load()
|
| 131 |
+
t = _topics(data).get(topic)
|
| 132 |
+
|
| 133 |
+
if t is None:
|
| 134 |
+
_topics(data)[topic] = {
|
| 135 |
+
"first": now, "last": now, "count": 1,
|
| 136 |
+
"acknowledged": False, "muted_until": 0, "snoozed_until": 0,
|
| 137 |
+
"importance": importance,
|
| 138 |
+
}
|
| 139 |
+
_save(data)
|
| 140 |
+
return True
|
| 141 |
+
|
| 142 |
+
if t.get("muted_until", 0) > now:
|
| 143 |
+
return False
|
| 144 |
+
if t.get("snoozed_until", 0) > now:
|
| 145 |
+
return False
|
| 146 |
+
if t.get("acknowledged"):
|
| 147 |
+
return False
|
| 148 |
+
if now - t.get("first", now) > MAX_ALERT_AGE:
|
| 149 |
+
return False
|
| 150 |
+
|
| 151 |
+
ladder = CRITICAL_LADDER if t.get("importance") == "critical" else REPEAT_LADDER
|
| 152 |
+
count = int(t.get("count", 1))
|
| 153 |
+
if count > len(ladder):
|
| 154 |
+
return False # said enough; wait to be asked
|
| 155 |
+
due = t.get("last", 0) + ladder[count - 1]
|
| 156 |
+
if now < due:
|
| 157 |
+
return False
|
| 158 |
+
|
| 159 |
+
t["last"] = now
|
| 160 |
+
t["count"] = count + 1
|
| 161 |
+
_save(data)
|
| 162 |
+
return True
|
| 163 |
+
|
| 164 |
+
|
| 165 |
+
def acknowledge(topic: str) -> None:
|
| 166 |
+
"""The user has dealt with it (or heard it). Stop raising it."""
|
| 167 |
+
with _lock:
|
| 168 |
+
data = _load()
|
| 169 |
+
t = _topics(data).setdefault(topic, {})
|
| 170 |
+
t["acknowledged"] = True
|
| 171 |
+
t["last"] = time.time()
|
| 172 |
+
_save(data)
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def snooze(topic: str, seconds: float) -> None:
|
| 176 |
+
"""Raise it again after `seconds`, then resume the normal ladder."""
|
| 177 |
+
with _lock:
|
| 178 |
+
data = _load()
|
| 179 |
+
t = _topics(data).setdefault(topic, {})
|
| 180 |
+
t["snoozed_until"] = time.time() + max(0.0, seconds)
|
| 181 |
+
t["acknowledged"] = False
|
| 182 |
+
# The repeat count resets: after an explicit "tell me in 30 minutes",
|
| 183 |
+
# the next mention is a fresh first mention, not a continuation of an
|
| 184 |
+
# escalation the user already responded to.
|
| 185 |
+
t["count"] = 1
|
| 186 |
+
_save(data)
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def mute(topic: str, seconds: float) -> None:
|
| 190 |
+
"""Say nothing about this topic at all for `seconds`."""
|
| 191 |
+
with _lock:
|
| 192 |
+
data = _load()
|
| 193 |
+
t = _topics(data).setdefault(topic, {})
|
| 194 |
+
t["muted_until"] = time.time() + max(0.0, seconds)
|
| 195 |
+
_save(data)
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
def reset(topic: str) -> None:
|
| 199 |
+
"""The condition cleared. Forget it, so a recurrence starts fresh."""
|
| 200 |
+
with _lock:
|
| 201 |
+
data = _load()
|
| 202 |
+
_topics(data).pop(topic, None)
|
| 203 |
+
_save(data)
|
| 204 |
+
|
| 205 |
+
|
| 206 |
+
def open_topics() -> list[dict]:
|
| 207 |
+
"""Alerts still waiting to be acknowledged — for 'what did I miss?'."""
|
| 208 |
+
now = time.time()
|
| 209 |
+
with _lock:
|
| 210 |
+
data = _load()
|
| 211 |
+
out = []
|
| 212 |
+
for name, t in _topics(data).items():
|
| 213 |
+
if t.get("acknowledged") or t.get("muted_until", 0) > now:
|
| 214 |
+
continue
|
| 215 |
+
out.append({"topic": name, "count": t.get("count", 1),
|
| 216 |
+
"first": t.get("first", 0),
|
| 217 |
+
"snoozed_until": t.get("snoozed_until", 0)})
|
| 218 |
+
return sorted(out, key=lambda d: d["first"])
|
| 219 |
+
|
| 220 |
+
|
| 221 |
+
# ── Natural language ─────────────────────────────────────────────────────────
|
| 222 |
+
|
| 223 |
+
_UNITS = {
|
| 224 |
+
"sec": 1, "secs": 1, "second": 1, "seconds": 1,
|
| 225 |
+
"min": 60, "mins": 60, "minute": 60, "minutes": 60,
|
| 226 |
+
"hr": 3600, "hrs": 3600, "hour": 3600, "hours": 3600,
|
| 227 |
+
"day": 86400, "days": 86400,
|
| 228 |
+
}
|
| 229 |
+
|
| 230 |
+
_WORD_NUMBERS = {
|
| 231 |
+
"a": 1, "an": 1, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5,
|
| 232 |
+
"six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, "fifteen": 15,
|
| 233 |
+
"twenty": 20, "thirty": 30, "forty": 40, "forty-five": 45, "sixty": 60,
|
| 234 |
+
"half": 0.5,
|
| 235 |
+
}
|
| 236 |
+
|
| 237 |
+
|
| 238 |
+
def parse_duration(text: str) -> Optional[float]:
|
| 239 |
+
"""Seconds from 'in 30 minutes', 'for 2 hours', 'half an hour'."""
|
| 240 |
+
lo = (text or "").lower()
|
| 241 |
+
|
| 242 |
+
m = re.search(r"half\s+an?\s+hour", lo)
|
| 243 |
+
if m:
|
| 244 |
+
return 1800.0
|
| 245 |
+
|
| 246 |
+
m = re.search(r"(\d+(?:\.\d+)?)\s*(" + "|".join(_UNITS) + r")\b", lo)
|
| 247 |
+
if m:
|
| 248 |
+
return float(m.group(1)) * _UNITS[m.group(2)]
|
| 249 |
+
|
| 250 |
+
words = "|".join(re.escape(w) for w in _WORD_NUMBERS)
|
| 251 |
+
m = re.search(r"\b(" + words + r")\s+(" + "|".join(_UNITS) + r")\b", lo)
|
| 252 |
+
if m:
|
| 253 |
+
return float(_WORD_NUMBERS[m.group(1)]) * _UNITS[m.group(2)]
|
| 254 |
+
return None
|
| 255 |
+
|
| 256 |
+
|
| 257 |
+
def parse_alert_command(text: str) -> Optional[dict]:
|
| 258 |
+
"""Turn a spoken instruction into an action on the open alerts.
|
| 259 |
+
|
| 260 |
+
Returns {"action": ..., "seconds": ..., "topic": ...} or None when the
|
| 261 |
+
sentence is not about alerts at all. `topic` is None for "this"/"that",
|
| 262 |
+
meaning the most recently raised alert.
|
| 263 |
+
|
| 264 |
+
Deliberately conservative: an unrecognised sentence returns None so it can
|
| 265 |
+
be handled as an ordinary command, rather than being swallowed here.
|
| 266 |
+
"""
|
| 267 |
+
lo = (text or "").strip().lower()
|
| 268 |
+
if not lo:
|
| 269 |
+
return None
|
| 270 |
+
|
| 271 |
+
secs = parse_duration(lo)
|
| 272 |
+
|
| 273 |
+
# "don't alert/tell/remind me about X for 2 hours"
|
| 274 |
+
if re.search(r"\b(don'?t|do not|stop|no more|quit)\b", lo) and \
|
| 275 |
+
re.search(r"\b(alert|alerts|tell|telling|remind|reminding|notify|"
|
| 276 |
+
r"notifying|warn|warning|mention)\b", lo):
|
| 277 |
+
topic = _extract_topic(lo)
|
| 278 |
+
return {"action": "mute", "seconds": secs or 3600.0, "topic": topic}
|
| 279 |
+
|
| 280 |
+
# "tell me in 30 minutes", "remind me in an hour", "not now"
|
| 281 |
+
if re.search(r"\b(later|not now|not right now|in a (bit|moment|while)|"
|
| 282 |
+
r"remind me|tell me|come back|ask me)\b", lo):
|
| 283 |
+
return {"action": "snooze", "seconds": secs or 1800.0,
|
| 284 |
+
"topic": _extract_topic(lo)}
|
| 285 |
+
|
| 286 |
+
# "ok", "got it", "noted", "I know", "acknowledged"
|
| 287 |
+
if re.fullmatch(r"(ok(ay)?|got it|noted|i know|understood|acknowledged|"
|
| 288 |
+
r"thanks|thank you|fine|sure|dismiss(ed)?)[.! ]*", lo):
|
| 289 |
+
return {"action": "acknowledge", "seconds": None, "topic": None}
|
| 290 |
+
|
| 291 |
+
# "what did I miss", "any alerts"
|
| 292 |
+
if re.search(r"\b(what did i miss|anything pending|any alerts|"
|
| 293 |
+
r"what'?s (pending|open|waiting))\b", lo):
|
| 294 |
+
return {"action": "list", "seconds": None, "topic": None}
|
| 295 |
+
|
| 296 |
+
return None
|
| 297 |
+
|
| 298 |
+
|
| 299 |
+
def _extract_topic(text: str) -> Optional[str]:
|
| 300 |
+
"""Pull the subject out of 'about the disk space' style phrasing."""
|
| 301 |
+
m = re.search(r"\babout (?:the |my |this |that )?([a-z0-9 _-]{2,40})", text)
|
| 302 |
+
if not m:
|
| 303 |
+
return None
|
| 304 |
+
topic = m.group(1).strip()
|
| 305 |
+
# Trim a trailing duration clause: "about the disk for two hours".
|
| 306 |
+
topic = re.split(r"\b(for|in|until|till)\b", topic)[0].strip()
|
| 307 |
+
return topic or None
|
modules/assistant_identity.py
CHANGED
|
@@ -1,6 +1,6 @@
|
|
| 1 |
import json
|
| 2 |
import os
|
| 3 |
-
from typing import Dict
|
| 4 |
|
| 5 |
from config import DATA_DIR
|
| 6 |
|
|
@@ -294,6 +294,414 @@ def get_identity_persona_prompt() -> str:
|
|
| 294 |
return base + _favorites_block(m)
|
| 295 |
|
| 296 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 297 |
def get_voice_id_for_mode(mode: str = None) -> str:
|
| 298 |
# Preferred built-in Edge voices for current setup.
|
| 299 |
# JARVIS target: closest practical style is British male.
|
|
|
|
| 1 |
import json
|
| 2 |
import os
|
| 3 |
+
from typing import Dict, Optional
|
| 4 |
|
| 5 |
from config import DATA_DIR
|
| 6 |
|
|
|
|
| 294 |
return base + _favorites_block(m)
|
| 295 |
|
| 296 |
|
| 297 |
+
# ── Judgement ────────────────────────────────────────────────────────────────
|
| 298 |
+
# The characterisation above says WHO he is. It says nothing about how to
|
| 299 |
+
# behave in a given moment, and that omission produced every complaint the
|
| 300 |
+
# operator raised: "at your service" prefixed to every reply, markdown asterisks
|
| 301 |
+
# in speech, a running commentary on research nobody asked about, and — worst —
|
| 302 |
+
# confident claims about the PC being connected while it sat powered off.
|
| 303 |
+
#
|
| 304 |
+
# The last one is not a style problem. The cloud chat path handed the model the
|
| 305 |
+
# characterisation and the user's sentence and nothing else, so when asked a
|
| 306 |
+
# question of fact about the system it had no facts, and filled the gap. An
|
| 307 |
+
# assistant that invents its own status is worse than one that says it does not
|
| 308 |
+
# know, because the operator cannot tell the difference until it matters.
|
| 309 |
+
#
|
| 310 |
+
# This block lives here, beside the identity, because it must reach BOTH brains.
|
| 311 |
+
# The previous version of this guidance lived inside core/brain.py, which is the
|
| 312 |
+
# desktop only — the phone talks to the cloud, the cloud calls token_manager,
|
| 313 |
+
# and token_manager never imports core.brain. That is why none of it was on the
|
| 314 |
+
# phone. One definition, imported in both places, is the only arrangement that
|
| 315 |
+
# does not drift.
|
| 316 |
+
JUDGEMENT_PROMPT = """
|
| 317 |
+
HOW TO BEHAVE IN THIS MOMENT
|
| 318 |
+
|
| 319 |
+
You are not filling in a form. Every one of the following is a judgement you
|
| 320 |
+
make fresh each turn, from what is actually in front of you.
|
| 321 |
+
|
| 322 |
+
1. ANSWER THE QUESTION FIRST.
|
| 323 |
+
Do not open with a greeting, an acknowledgement, or a readiness phrase and
|
| 324 |
+
then answer. "At your service" is an answer to "are you there?" — it is not
|
| 325 |
+
a preface to other answers. Asked whether the PC is connected, the first
|
| 326 |
+
word out is yes or no. Asked who someone is, the first word is about them.
|
| 327 |
+
If you catch yourself writing a courtesy line before the substance, delete
|
| 328 |
+
it and start at the substance.
|
| 329 |
+
|
| 330 |
+
2. LENGTH IS JUDGED, NEVER FIXED.
|
| 331 |
+
Match the weight of the reply to the weight of the question.
|
| 332 |
+
- "You there?" -> "At your service." Nothing after it.
|
| 333 |
+
- "Is the PC connected?" -> "No, it's offline." Nothing after it.
|
| 334 |
+
- A real question -> answer it, and only it.
|
| 335 |
+
- Something genuinely complex, or an explanation you were asked for ->
|
| 336 |
+
take the room you need. Brevity is the absence of padding, not a word
|
| 337 |
+
limit; a long answer that is all substance is correct.
|
| 338 |
+
Never append a status report, a summary of what you just did, an offer of
|
| 339 |
+
further help, or a list of what else you could do. Never restate the
|
| 340 |
+
request back. Never end with "let me know if you need anything else".
|
| 341 |
+
|
| 342 |
+
3. WORK OUT THE SITUATION FROM THE EVIDENCE, THEN ACT ON IT.
|
| 343 |
+
The LIVE STATE below carries raw readings from the operator's phone — the
|
| 344 |
+
time, roughly where they are, how fast they are moving, which app is open,
|
| 345 |
+
whether a calendar event is running, what is connected by Bluetooth, whether
|
| 346 |
+
the ringer is silent, whether headphones are in, what the wifi network is
|
| 347 |
+
called. They are readings, not labels. Nothing tells you "he is in a
|
| 348 |
+
meeting" or "she is driving"; you infer it, the way a person in the room
|
| 349 |
+
would, from what the readings add up to.
|
| 350 |
+
|
| 351 |
+
Do that inference every turn, and let it change what you do:
|
| 352 |
+
- Who is likely around, and therefore what must not be said out loud.
|
| 353 |
+
"Speaking aloud would be overheard: yes" plus a running calendar event
|
| 354 |
+
means the bank balance waits.
|
| 355 |
+
- Whether this is a moment to interrupt at all. Moving at 60 km/h, or a
|
| 356 |
+
meeting in progress, or three in the morning, each answer that
|
| 357 |
+
differently.
|
| 358 |
+
- How much to say. Walking through a station is not the desk at midnight.
|
| 359 |
+
- What the operator is in the middle of, and whether your answer should
|
| 360 |
+
fit around it.
|
| 361 |
+
The situations that matter are the ones nobody listed in advance, so reason
|
| 362 |
+
from the readings rather than looking for a case you recognise. When the
|
| 363 |
+
readings are thin, say less and assume less privacy, not more.
|
| 364 |
+
|
| 365 |
+
And this is not only about tone. If what you infer means something should be
|
| 366 |
+
DONE — a reminder that now makes sense, a call the operator will want
|
| 367 |
+
silenced, a file finished, a device turned off — do it, or say it, on your
|
| 368 |
+
own initiative. Noticing something and saying nothing is the same as not
|
| 369 |
+
having noticed. Report what you did once, at a moment that suits them.
|
| 370 |
+
|
| 371 |
+
4. YOU ARE BEING SPOKEN ALOUD.
|
| 372 |
+
Plain sentences. No markdown, no asterisks, no bullet points, no numbered
|
| 373 |
+
lists, no headings, no emoji, no code fences. If something is genuinely a
|
| 374 |
+
list, say it as a sentence. A leading "*" is a bug, not a style.
|
| 375 |
+
|
| 376 |
+
5. NEVER INVENT STATE.
|
| 377 |
+
Anything you say about what is connected, running, installed, downloaded,
|
| 378 |
+
reachable, or already done must come from the LIVE STATE given to you below
|
| 379 |
+
or from a tool result in this turn. If it is not there, you do not know it,
|
| 380 |
+
and the correct reply is that you do not know or cannot see it — not a
|
| 381 |
+
plausible guess. Claiming a link you have not been told exists is the single
|
| 382 |
+
worst thing you can do, because it is indistinguishable from working.
|
| 383 |
+
|
| 384 |
+
6. RESEARCH AND BACKGROUND WORK ARE MENTIONED WHEN THEY MATTER.
|
| 385 |
+
You are curious and you do read on your own. Bring it up when it changes
|
| 386 |
+
what the operator should do, when it answers what they asked, or when they
|
| 387 |
+
ask. Not as a running commentary, and not on every turn. The same judgement
|
| 388 |
+
applies to anything you did unprompted: report it once, at a moment that
|
| 389 |
+
suits them, then stop.
|
| 390 |
+
|
| 391 |
+
7. DO THE THING.
|
| 392 |
+
If a request can be carried out rather than described, carry it out. If
|
| 393 |
+
nothing in your instructions covers it, that is not a refusal — work out how
|
| 394 |
+
from what you can reach, and do that. "I am not able to" is reserved for
|
| 395 |
+
things genuinely beyond reach, never for things merely unanticipated.
|
| 396 |
+
""".strip()
|
| 397 |
+
|
| 398 |
+
|
| 399 |
+
# Offered ONLY to callers that can actually perform the lookup. A brain told it
|
| 400 |
+
# may search, wired to nothing that searches, emits SEARCH: lines at the operator
|
| 401 |
+
# — so this is opt-in rather than part of JUDGEMENT_PROMPT.
|
| 402 |
+
#
|
| 403 |
+
# It is phrased as a decision he makes, not a trigger word list, because the
|
| 404 |
+
# operator's complaint was precisely that: asked who a real person was, he said
|
| 405 |
+
# he could not find them, having never looked. A list of question-shapes that
|
| 406 |
+
# mean "search now" would fail the same way on the first shape nobody listed.
|
| 407 |
+
LOOKUP_PROTOCOL = """
|
| 408 |
+
LOOKING THINGS UP
|
| 409 |
+
|
| 410 |
+
You can search the web and read pages. Use it whenever answering properly needs
|
| 411 |
+
something you do not reliably know: who a particular person is, what happened
|
| 412 |
+
recently, a current price, a specific fact about a named thing. You are not
|
| 413 |
+
expected to know these from memory, and guessing at them is worse than looking.
|
| 414 |
+
|
| 415 |
+
To search, make the ENTIRE reply exactly one line:
|
| 416 |
+
|
| 417 |
+
SEARCH: <the query>
|
| 418 |
+
|
| 419 |
+
Nothing else on that line and nothing after it. The results come straight back
|
| 420 |
+
to you and you then answer normally. You may search again if the first results
|
| 421 |
+
were not enough, up to a few times.
|
| 422 |
+
|
| 423 |
+
Judge it. A question about the operator's own machine, a greeting, an opinion,
|
| 424 |
+
a calculation, or anything you genuinely know does not need a search. "I could
|
| 425 |
+
not find anything" is only ever true AFTER you have looked.
|
| 426 |
+
""".strip()
|
| 427 |
+
|
| 428 |
+
|
| 429 |
+
# Offered ONLY to a caller that can actually perform the read on the device.
|
| 430 |
+
#
|
| 431 |
+
# Hands without eyes: he could send a reply and not read the message he was
|
| 432 |
+
# replying to, open the camera and not see a photo, act on an instruction and
|
| 433 |
+
# never notice the notification that should have prompted it. Reading closes
|
| 434 |
+
# that, and it is what turns "do what I say" into "notice, then do".
|
| 435 |
+
READ_PROTOCOL = """
|
| 436 |
+
LOOKING AT THE PHONE ITSELF
|
| 437 |
+
|
| 438 |
+
You can read what is actually on the operator's phone — the text on screen right
|
| 439 |
+
now, their notifications, their messages and the content of them, the call log,
|
| 440 |
+
a contact, the clipboard, the calendar. Use it whenever answering or acting well
|
| 441 |
+
needs something only the phone knows: "reply to her" means reading what she
|
| 442 |
+
said first; "who called" means the call log; "what did I miss" means the
|
| 443 |
+
notifications.
|
| 444 |
+
|
| 445 |
+
To read, make the WHOLE reply exactly one line:
|
| 446 |
+
|
| 447 |
+
READ: <source> [| <filter>]
|
| 448 |
+
|
| 449 |
+
where source is one of:
|
| 450 |
+
screen the text visible on screen at this moment
|
| 451 |
+
notifications what is in the notification shade
|
| 452 |
+
sms recent texts and their content; filter by a name or a number
|
| 453 |
+
calls the call log; filter by a name
|
| 454 |
+
contacts look someone up; filter is REQUIRED (the name)
|
| 455 |
+
clipboard what is currently copied
|
| 456 |
+
calendar the next few days
|
| 457 |
+
|
| 458 |
+
for example:
|
| 459 |
+
READ: notifications
|
| 460 |
+
READ: sms | Anand
|
| 461 |
+
READ: contacts | mum
|
| 462 |
+
|
| 463 |
+
Nothing else on that line. What is read comes straight back to you and you then
|
| 464 |
+
answer or act on it. You may read more than once — read her message, then send
|
| 465 |
+
the reply — and a read can be the first step of a chain that ends in an action.
|
| 466 |
+
|
| 467 |
+
Only read what the turn actually needs, and remember the reading is private to
|
| 468 |
+
the operator: your judgement about whether speech would be overheard (it is in
|
| 469 |
+
LIVE STATE) decides whether you say what you read out loud or keep it on screen.
|
| 470 |
+
""".strip()
|
| 471 |
+
|
| 472 |
+
|
| 473 |
+
# Offered ONLY to callers whose reply is parsed for actions and executed.
|
| 474 |
+
#
|
| 475 |
+
# The assistant had hands the entire time and did not know it. The handset can
|
| 476 |
+
# already carry out forty-four kinds of action, drive any app through the
|
| 477 |
+
# accessibility service like a person would, plan arbitrary Android intents on
|
| 478 |
+
# device, and hand shell or freshly-written Python to the PC. The chat path was
|
| 479 |
+
# never told, so it answered "I cannot access your messages" while holding
|
| 480 |
+
# SEND_SMS and SEND_WHATSAPP — a refusal that was simply false.
|
| 481 |
+
#
|
| 482 |
+
# This is the difference between an assistant that discusses tasks and one that
|
| 483 |
+
# performs them, and it is what makes uncoded requests possible: there is a
|
| 484 |
+
# deliberate escape hatch for anything nobody wrote a handler for.
|
| 485 |
+
ACT_PROTOCOL = """
|
| 486 |
+
DOING THINGS, NOT DESCRIBING THEM
|
| 487 |
+
|
| 488 |
+
You are not a chat window. You are attached to the operator's phone and, when it
|
| 489 |
+
is reachable, their PC, and you can act on both. If a request can be carried out,
|
| 490 |
+
carry it out in the same turn rather than explaining how they might.
|
| 491 |
+
|
| 492 |
+
To act, make the WHOLE reply this JSON and nothing else — no prose around it, no
|
| 493 |
+
code fence:
|
| 494 |
+
|
| 495 |
+
{"message":"<the short line you say>","actions":[{"type":"<TYPE>","label":"<what it is>","payload":"<argument>"}]}
|
| 496 |
+
|
| 497 |
+
Types you can use, with their payload:
|
| 498 |
+
SEND_SMS "recipient|the message text"
|
| 499 |
+
SEND_WHATSAPP "recipient|the message text"
|
| 500 |
+
CALL_DIAL the number or contact name
|
| 501 |
+
OPEN_APP app name or package
|
| 502 |
+
OPEN_URL a full URL
|
| 503 |
+
OPEN_WEB_SEARCH the query
|
| 504 |
+
OPEN_YOUTUBE_SEARCH / OPEN_SPOTIFY_SEARCH the query
|
| 505 |
+
SET_ALARM "HH:MM|label"
|
| 506 |
+
SET_TIMER "seconds|label"
|
| 507 |
+
ADD_CALENDAR_EVENT "title|YYYY-MM-DD HH:MM|minutes|notes"
|
| 508 |
+
NAVIGATE_TO the destination
|
| 509 |
+
TORCH_ON / TORCH_OFF, SET_BRIGHTNESS (0-100), SET_DND ("on"/"off"),
|
| 510 |
+
WIFI_TOGGLE / BLUETOOTH_TOGGLE ("on"/"off"),
|
| 511 |
+
SET_MEDIA_VOLUME (0-100), ADJUST_MEDIA_VOLUME ("up"/"down"),
|
| 512 |
+
OPEN_CAMERA, SHARE_TEXT, OPEN_MARKET, START_GUARDIAN / STOP_GUARDIAN,
|
| 513 |
+
OPEN_WIFI_SETTINGS / OPEN_BLUETOOTH_SETTINGS / OPEN_LOCATION_SETTINGS /
|
| 514 |
+
OPEN_SOUND_SETTINGS / OPEN_DISPLAY_SETTINGS / OPEN_BATTERY_SETTINGS /
|
| 515 |
+
OPEN_ANDROID_SETTINGS / OPEN_NOTIFICATION_SETTINGS
|
| 516 |
+
|
| 517 |
+
RAW_INTENT payload is JSON describing any Android intent:
|
| 518 |
+
{"action":"android.intent.action.SEND","data":"mailto:x@y.com",
|
| 519 |
+
"type":"text/plain","package":"com.google.android.gm",
|
| 520 |
+
"extras":{"android.intent.extra.SUBJECT":"...","android.intent.extra.TEXT":"..."}}
|
| 521 |
+
Android expresses almost every capability of the device this way, so this
|
| 522 |
+
reaches things with no dedicated type above — composing an email, locking
|
| 523 |
+
the screen, taking a screenshot, toggling mobile data, controlling
|
| 524 |
+
playback, sharing a specific file, opening a settings panel nobody listed.
|
| 525 |
+
Reach for it before ever saying you cannot.
|
| 526 |
+
|
| 527 |
+
AUTOMATE_MOBILE_TASK payload is the task in plain words.
|
| 528 |
+
THIS IS THE ONE THAT MATTERS. It drives the screen through the
|
| 529 |
+
accessibility service — reading what is displayed and tapping, typing and
|
| 530 |
+
scrolling exactly as a person would. Anything a human could do on this
|
| 531 |
+
phone is reachable through it, whether or not a dedicated type exists:
|
| 532 |
+
"reply to Anand's last WhatsApp saying I'm driving, I'll call back",
|
| 533 |
+
"cancel my Swiggy order", "download and install that APK and open it",
|
| 534 |
+
"turn off every alarm for tomorrow". When nothing above fits, use this
|
| 535 |
+
rather than declining.
|
| 536 |
+
|
| 537 |
+
Chain freely — several actions run in order, so "text him and set a reminder and
|
| 538 |
+
turn the torch on" is one reply with three entries.
|
| 539 |
+
|
| 540 |
+
THE CHAIN IS YOURS TO INVENT, AND IT DOES NOT STOP AT ONE STEP.
|
| 541 |
+
|
| 542 |
+
Nobody has written the sequence for you. You compose it, step by step, out of
|
| 543 |
+
whatever the request actually needs, and you may mix every power you have in a
|
| 544 |
+
single piece of work:
|
| 545 |
+
|
| 546 |
+
look it up (SEARCH:) -> read what is on the phone (READ:) -> do something
|
| 547 |
+
(actions) -> see whether it worked -> do the next thing -> hand the rest to
|
| 548 |
+
the desktop (PC:) -> tell them what happened.
|
| 549 |
+
|
| 550 |
+
In any order, as many times as it takes. After you act, what ACTUALLY happened
|
| 551 |
+
on the device comes back to you — which step worked and which did nothing — and
|
| 552 |
+
you get to continue from there. So a step that fails is not the end of the
|
| 553 |
+
request; it is information. Try it a different way.
|
| 554 |
+
|
| 555 |
+
Two rules for the chain, and they matter more than the steps:
|
| 556 |
+
|
| 557 |
+
A step with no matching action type is NOT a dead end. RAW_INTENT lets you
|
| 558 |
+
compose any Android intent yourself, and the desktop engine writes new code
|
| 559 |
+
when nothing exists. Between them there is almost nothing the request can ask
|
| 560 |
+
for that has no route at all. "Nobody built that" is never the reason.
|
| 561 |
+
|
| 562 |
+
Never report a step as done because you asked for it. You will be told what
|
| 563 |
+
really happened. Say that, and only that. A chain that half worked, described
|
| 564 |
+
honestly, is worth more than one you claim completed.
|
| 565 |
+
|
| 566 |
+
WHEN NOTHING ABOVE IS ENOUGH, HAND IT TO THE MACHINE.
|
| 567 |
+
|
| 568 |
+
If the PC is connected, you have an engine there with no capability list at all.
|
| 569 |
+
It plans a request into steps, runs shell commands, and WRITES AND EXECUTES NEW
|
| 570 |
+
CODE OF ITS OWN when no existing tool fits. It repairs a step that fails and
|
| 571 |
+
tries again. That is the answer to anything genuinely unanticipated — not a
|
| 572 |
+
refusal, and not an apology.
|
| 573 |
+
|
| 574 |
+
To use it, make the ENTIRE reply exactly one line:
|
| 575 |
+
|
| 576 |
+
PC: <the task in plain words, including any chain>
|
| 577 |
+
|
| 578 |
+
for example:
|
| 579 |
+
PC: find every screenshot on the desktop from this week, put them in one PDF, open it
|
| 580 |
+
PC: download the latest release of that repo, install it, run it for ten minutes, then close it
|
| 581 |
+
PC: work out why the build is failing and fix it
|
| 582 |
+
|
| 583 |
+
Nothing else on that line and nothing after it. What happened comes back to you
|
| 584 |
+
and you then tell the operator, briefly. Irreversible work is deliberated over
|
| 585 |
+
and held with a window to cancel before it runs, so you may ask for it plainly
|
| 586 |
+
rather than refusing on their behalf.
|
| 587 |
+
|
| 588 |
+
Use it when the phone cannot do the thing, or when the work belongs on the
|
| 589 |
+
computer. Do not use it for something the phone handles perfectly well, and do
|
| 590 |
+
not use it when the PC is offline — LIVE STATE says which.
|
| 591 |
+
|
| 592 |
+
AND ACT ON YOUR OWN JUDGEMENT, NOT ONLY ON INSTRUCTIONS.
|
| 593 |
+
Everything above is equally available when nobody asked. If you notice something
|
| 594 |
+
that needs doing — work left unsaved, a download half finished, an alarm that
|
| 595 |
+
will fire in a meeting, a machine about to run out of power — do it, then
|
| 596 |
+
mention it once, later, at a moment that suits them. Waiting to be told is not
|
| 597 |
+
caution, it is absence. The only things that wait for permission are the
|
| 598 |
+
irreversible ones and the expensive ones.
|
| 599 |
+
|
| 600 |
+
Judgement still applies. Something irreversible, something that spends money, or
|
| 601 |
+
something the operator may not have meant — say what you are about to do and let
|
| 602 |
+
them confirm, rather than either doing it silently or refusing outright. And read
|
| 603 |
+
the moment: acting is usually better than asking, but not while they are driving
|
| 604 |
+
and not in front of company.
|
| 605 |
+
|
| 606 |
+
"I can't do that" is for things genuinely out of reach — no network, hardware
|
| 607 |
+
that is not there, a thing that does not exist. It is NEVER the answer to
|
| 608 |
+
something merely unanticipated. If no type fits, that is what
|
| 609 |
+
AUTOMATE_MOBILE_TASK is for, and if even that will not reach it, say precisely
|
| 610 |
+
what stops you rather than a flat refusal.
|
| 611 |
+
""".strip()
|
| 612 |
+
|
| 613 |
+
|
| 614 |
+
def _live_state_block(live_state: Optional[Dict[str, object]]) -> str:
|
| 615 |
+
"""Real, measured facts about the system, rendered for the model.
|
| 616 |
+
|
| 617 |
+
Rule 5 above is unenforceable without this. The point is not only to supply
|
| 618 |
+
what is true but to make the ABSENCE of a fact visible: a key that is
|
| 619 |
+
present with an "unknown" value is far less likely to be confabulated than
|
| 620 |
+
a key that was never mentioned.
|
| 621 |
+
"""
|
| 622 |
+
if not live_state:
|
| 623 |
+
return (
|
| 624 |
+
"\n\nLIVE STATE\nNothing was measured for this turn. You therefore "
|
| 625 |
+
"know nothing about what is connected or running, and must say so "
|
| 626 |
+
"plainly if asked rather than guessing."
|
| 627 |
+
)
|
| 628 |
+
lines = []
|
| 629 |
+
for key, value in live_state.items():
|
| 630 |
+
if value is None or value == "":
|
| 631 |
+
value = "unknown"
|
| 632 |
+
lines.append(f"- {key}: {value}")
|
| 633 |
+
return (
|
| 634 |
+
"\n\nLIVE STATE (measured just now — the ONLY system facts you have; "
|
| 635 |
+
"anything not listed here is unknown to you)\n" + "\n".join(lines)
|
| 636 |
+
)
|
| 637 |
+
|
| 638 |
+
|
| 639 |
+
def compose_identity(
|
| 640 |
+
persona: Optional[str] = None,
|
| 641 |
+
live_state: Optional[Dict[str, object]] = None,
|
| 642 |
+
include_judgement: bool = True,
|
| 643 |
+
can_search: bool = False,
|
| 644 |
+
can_act: bool = False,
|
| 645 |
+
can_read: bool = False,
|
| 646 |
+
) -> str:
|
| 647 |
+
"""The complete system prompt: who he is, how to behave, what is true.
|
| 648 |
+
|
| 649 |
+
This is what every conversational caller should use — desktop brain, cloud
|
| 650 |
+
chat, phone. Callers that need only the characterisation (a voice sample, a
|
| 651 |
+
persona preview) can still reach for get_identity_persona_prompt().
|
| 652 |
+
|
| 653 |
+
`can_search` must only be set by a caller that actually runs the SEARCH:
|
| 654 |
+
line it will get back.
|
| 655 |
+
"""
|
| 656 |
+
p = (persona or get_mode() or "friday").strip().lower()
|
| 657 |
+
if p not in ("jarvis", "friday"):
|
| 658 |
+
p = "friday"
|
| 659 |
+
base = JARVIS_PERSONALITY_PROMPT if p == "jarvis" else FRIDAY_PERSONALITY_PROMPT
|
| 660 |
+
out = base + _favorites_block(p)
|
| 661 |
+
if include_judgement:
|
| 662 |
+
out += "\n\n" + JUDGEMENT_PROMPT
|
| 663 |
+
if can_search:
|
| 664 |
+
out += "\n\n" + LOOKUP_PROTOCOL
|
| 665 |
+
if can_act:
|
| 666 |
+
out += "\n\n" + ACT_PROTOCOL
|
| 667 |
+
if can_read:
|
| 668 |
+
out += "\n\n" + READ_PROTOCOL
|
| 669 |
+
out += _live_state_block(live_state)
|
| 670 |
+
return out
|
| 671 |
+
|
| 672 |
+
|
| 673 |
+
# ── Spoken output ────────────────────────────────────────────────────────────
|
| 674 |
+
_MARKDOWN_NOISE = (
|
| 675 |
+
(r"```[a-zA-Z0-9_+-]*\n?", ""), # code fences
|
| 676 |
+
(r"^\s{0,3}#{1,6}\s+", ""), # headings
|
| 677 |
+
(r"^\s{0,3}[-*+]\s+", ""), # bullet markers
|
| 678 |
+
(r"^\s{0,3}>\s?", ""), # block quotes
|
| 679 |
+
(r"\*\*(.+?)\*\*", r"\1"), # bold
|
| 680 |
+
(r"(?<!\w)\*(?!\s)(.+?)(?<!\s)\*(?!\w)", r"\1"), # italics
|
| 681 |
+
(r"(?<!\w)_(?!\s)(.+?)(?<!\s)_(?!\w)", r"\1"), # underscore italics
|
| 682 |
+
(r"`([^`]+)`", r"\1"), # inline code
|
| 683 |
+
)
|
| 684 |
+
|
| 685 |
+
|
| 686 |
+
def to_spoken(text: str) -> str:
|
| 687 |
+
"""Strip formatting from a reply that will be read aloud.
|
| 688 |
+
|
| 689 |
+
The prompt already tells him not to write markdown, and mostly he will not.
|
| 690 |
+
This is the belt to that braces: an LLM under a long system prompt reverts to
|
| 691 |
+
bullet points under load, and a leading asterisk read aloud is a defect the
|
| 692 |
+
operator hears every single time. Cheap, deterministic, and it cannot make a
|
| 693 |
+
correct reply wrong.
|
| 694 |
+
"""
|
| 695 |
+
import re
|
| 696 |
+
if not text:
|
| 697 |
+
return ""
|
| 698 |
+
out = text
|
| 699 |
+
for pattern, repl in _MARKDOWN_NOISE:
|
| 700 |
+
out = re.sub(pattern, repl, out, flags=re.MULTILINE)
|
| 701 |
+
out = re.sub(r"\n{3,}", "\n\n", out)
|
| 702 |
+
return out.strip()
|
| 703 |
+
|
| 704 |
+
|
| 705 |
def get_voice_id_for_mode(mode: str = None) -> str:
|
| 706 |
# Preferred built-in Edge voices for current setup.
|
| 707 |
# JARVIS target: closest practical style is British male.
|
modules/deferred_intent.py
ADDED
|
@@ -0,0 +1,268 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""modules/deferred_intent.py — thinking hard before an irreversible act,
|
| 2 |
+
and then remembering that it was ordered.
|
| 3 |
+
|
| 4 |
+
THE RULE THIS IMPLEMENTS
|
| 5 |
+
------------------------
|
| 6 |
+
"if the command said to do shutdown or anything jarvis will think about that
|
| 7 |
+
very deeply and if didn't shutdown for a reason and even after some minutes no
|
| 8 |
+
response then think again if reason comes to shutdown or anything i said he
|
| 9 |
+
will do that"
|
| 10 |
+
|
| 11 |
+
Which is a specific and quite subtle behaviour:
|
| 12 |
+
|
| 13 |
+
1. An irreversible order is DELIBERATED, not just executed on sight.
|
| 14 |
+
2. If there is a real reason to hold — a download mid-flight, unsaved work,
|
| 15 |
+
a backup running — JARVIS holds, and SAYS WHY.
|
| 16 |
+
3. Holding is not refusing. The order stands.
|
| 17 |
+
4. After a few minutes with no word from the operator, he thinks again.
|
| 18 |
+
5. If the reason has cleared, or on reflection it no longer outweighs the
|
| 19 |
+
instruction, HE DOES IT. What was asked for happens.
|
| 20 |
+
|
| 21 |
+
The failure mode being avoided in both directions matters. A machine that shuts
|
| 22 |
+
down mid-download because it heard a word is bad. A machine that quietly drops
|
| 23 |
+
an order it was given because it once had a doubt is worse — the operator
|
| 24 |
+
believes the thing is happening and it never does. So a deferral is a TIMER,
|
| 25 |
+
never a bin.
|
| 26 |
+
|
| 27 |
+
State is on disk: a deferral that does not survive a process restart is exactly
|
| 28 |
+
the silent drop this is designed to prevent.
|
| 29 |
+
"""
|
| 30 |
+
|
| 31 |
+
from __future__ import annotations
|
| 32 |
+
|
| 33 |
+
import json
|
| 34 |
+
import os
|
| 35 |
+
import threading
|
| 36 |
+
import time
|
| 37 |
+
from dataclasses import dataclass
|
| 38 |
+
from typing import Callable, Optional
|
| 39 |
+
|
| 40 |
+
_STATE_DIR = os.environ.get("JARVIS_APP_DATA_DIR") or os.path.join(
|
| 41 |
+
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".friday_data")
|
| 42 |
+
_STATE_PATH = os.path.join(_STATE_DIR, "deferred_orders.json")
|
| 43 |
+
|
| 44 |
+
_lock = threading.RLock()
|
| 45 |
+
|
| 46 |
+
# How long to hold before reconsidering, per attempt. It reconsiders sooner the
|
| 47 |
+
# first time — "a few minutes" — then gives the blocking condition longer to
|
| 48 |
+
# clear before deciding it is not going to.
|
| 49 |
+
RECONSIDER_LADDER = (180, 300, 600, 900)
|
| 50 |
+
|
| 51 |
+
# After this many reconsiderations the order is carried out regardless of the
|
| 52 |
+
# standing objection, because at that point the objection has had the better
|
| 53 |
+
# part of half an hour to resolve itself and the operator has said nothing to
|
| 54 |
+
# withdraw the instruction.
|
| 55 |
+
MAX_DEFERRALS = len(RECONSIDER_LADDER)
|
| 56 |
+
|
| 57 |
+
|
| 58 |
+
@dataclass
|
| 59 |
+
class Verdict:
|
| 60 |
+
proceed: bool
|
| 61 |
+
reason: str
|
| 62 |
+
speak: str
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def _load() -> dict:
|
| 66 |
+
try:
|
| 67 |
+
with open(_STATE_PATH, "r", encoding="utf-8") as fh:
|
| 68 |
+
data = json.load(fh)
|
| 69 |
+
return data if isinstance(data, dict) else {}
|
| 70 |
+
except Exception:
|
| 71 |
+
return {}
|
| 72 |
+
|
| 73 |
+
|
| 74 |
+
def _save(data: dict) -> None:
|
| 75 |
+
try:
|
| 76 |
+
os.makedirs(_STATE_DIR, exist_ok=True)
|
| 77 |
+
tmp = _STATE_PATH + ".tmp"
|
| 78 |
+
with open(tmp, "w", encoding="utf-8") as fh:
|
| 79 |
+
json.dump(data, fh, indent=2)
|
| 80 |
+
os.replace(tmp, _STATE_PATH)
|
| 81 |
+
except Exception:
|
| 82 |
+
pass
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _orders(data: dict) -> dict:
|
| 86 |
+
return data.setdefault("orders", {})
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
# ── What is actually going on right now ─────────────────────────────────────
|
| 90 |
+
def machine_situation() -> str:
|
| 91 |
+
"""Facts a person would check before pulling the plug."""
|
| 92 |
+
bits = []
|
| 93 |
+
try:
|
| 94 |
+
import psutil
|
| 95 |
+
cpu = psutil.cpu_percent(interval=0.4)
|
| 96 |
+
bits.append(f"CPU {cpu:.0f}%")
|
| 97 |
+
try:
|
| 98 |
+
io1 = psutil.net_io_counters()
|
| 99 |
+
time.sleep(0.6)
|
| 100 |
+
io2 = psutil.net_io_counters()
|
| 101 |
+
kbs = (io2.bytes_recv - io1.bytes_recv) / 0.6 / 1024.0
|
| 102 |
+
bits.append(f"network in {kbs:.0f} KB/s")
|
| 103 |
+
except Exception:
|
| 104 |
+
pass
|
| 105 |
+
busy = []
|
| 106 |
+
for p in psutil.process_iter(["name", "cpu_percent"]):
|
| 107 |
+
try:
|
| 108 |
+
n = (p.info.get("name") or "").lower()
|
| 109 |
+
if any(k in n for k in ("setup", "install", "msiexec", "update",
|
| 110 |
+
"backup", "7z", "winrar", "ffmpeg",
|
| 111 |
+
"handbrake", "robocopy", "git")):
|
| 112 |
+
busy.append(n)
|
| 113 |
+
except Exception:
|
| 114 |
+
continue
|
| 115 |
+
if busy:
|
| 116 |
+
bits.append("running: " + ", ".join(sorted(set(busy))[:6]))
|
| 117 |
+
try:
|
| 118 |
+
b = psutil.sensors_battery()
|
| 119 |
+
if b:
|
| 120 |
+
bits.append(f"battery {b.percent:.0f}%"
|
| 121 |
+
+ ("" if b.power_plugged else ", on battery"))
|
| 122 |
+
except Exception:
|
| 123 |
+
pass
|
| 124 |
+
except Exception:
|
| 125 |
+
bits.append("(process inspection unavailable)")
|
| 126 |
+
return "; ".join(bits) or "nothing notable"
|
| 127 |
+
|
| 128 |
+
|
| 129 |
+
_DELIBERATE = """An irreversible instruction has been given by the operator.
|
| 130 |
+
It WILL be carried out — you are deciding whether NOW is the moment, not
|
| 131 |
+
whether to obey.
|
| 132 |
+
|
| 133 |
+
Reply with only JSON:
|
| 134 |
+
{"proceed": true|false, "reason": "<one short clause>",
|
| 135 |
+
"speak": "<what to say aloud, one sentence>"}
|
| 136 |
+
|
| 137 |
+
Hold (proceed=false) ONLY for a concrete condition visible in the situation
|
| 138 |
+
that the act would damage: an install or download mid-flight, a backup or long
|
| 139 |
+
encode running, work that would be lost. Say which one in `reason`.
|
| 140 |
+
|
| 141 |
+
Do NOT hold because the action is significant, or to be careful, or to seek
|
| 142 |
+
confirmation. The operator already decided. Absent a real conflict, proceed.
|
| 143 |
+
"""
|
| 144 |
+
|
| 145 |
+
|
| 146 |
+
def deliberate(order: str, attempt: int = 0) -> Verdict:
|
| 147 |
+
"""Think about it properly, AS HIMSELF.
|
| 148 |
+
|
| 149 |
+
The judgement is prefixed with the live persona from
|
| 150 |
+
modules/assistant_identity.py rather than being made by a neutral
|
| 151 |
+
classifier. Whether something is worth interrupting the operator over is a
|
| 152 |
+
question of character, not of policy — JARVIS weighs it differently from
|
| 153 |
+
FRIDAY, and both of them weigh it differently from a generic safety filter.
|
| 154 |
+
The line he speaks has to sound like him too, since it is the same voice
|
| 155 |
+
that says everything else.
|
| 156 |
+
|
| 157 |
+
Falls open — toward obeying — if the model is unreachable.
|
| 158 |
+
"""
|
| 159 |
+
situation = machine_situation()
|
| 160 |
+
ctx = (f"ORDER: {order}\nSITUATION: {situation}\n"
|
| 161 |
+
f"TIMES ALREADY DEFERRED: {attempt}")
|
| 162 |
+
if attempt >= 1:
|
| 163 |
+
ctx += ("\nNOTE: this was held back before and the operator has said "
|
| 164 |
+
"nothing since. Absent a still-active conflict, carry it out.")
|
| 165 |
+
|
| 166 |
+
system = _DELIBERATE
|
| 167 |
+
try:
|
| 168 |
+
from modules.assistant_identity import get_identity_persona_prompt
|
| 169 |
+
identity = (get_identity_persona_prompt() or "").strip()
|
| 170 |
+
if identity:
|
| 171 |
+
system = (f"{identity}\n\n---\n"
|
| 172 |
+
f"You are deciding about an order you have just been "
|
| 173 |
+
f"given.\n{_DELIBERATE}")
|
| 174 |
+
except Exception:
|
| 175 |
+
pass
|
| 176 |
+
|
| 177 |
+
try:
|
| 178 |
+
from core.brain_router import ask_router
|
| 179 |
+
rep = ask_router(user_text=order, system_prompt=system,
|
| 180 |
+
context_block=ctx, provider_order=["gemini", "local"])
|
| 181 |
+
raw = (getattr(rep, "text", "") or "").strip()
|
| 182 |
+
import re
|
| 183 |
+
raw = re.sub(r"^```(?:json)?|```$", "", raw).strip()
|
| 184 |
+
m = re.search(r"\{.*\}", raw, re.S)
|
| 185 |
+
if m:
|
| 186 |
+
d = json.loads(m.group(0))
|
| 187 |
+
return Verdict(bool(d.get("proceed", True)),
|
| 188 |
+
str(d.get("reason", ""))[:200],
|
| 189 |
+
str(d.get("speak", ""))[:300])
|
| 190 |
+
except Exception:
|
| 191 |
+
pass
|
| 192 |
+
# No model, or unparseable: the instruction stands. Refusing to act because
|
| 193 |
+
# the reasoning layer is down would be the silent drop this module exists
|
| 194 |
+
# to prevent.
|
| 195 |
+
return Verdict(True, "no blocking condition found", "")
|
| 196 |
+
|
| 197 |
+
|
| 198 |
+
# ── Deferral bookkeeping ────────────────────────────────────────────────────
|
| 199 |
+
def defer(order: str, reason: str) -> int:
|
| 200 |
+
"""Record a held order. Returns how many times it has now been held."""
|
| 201 |
+
with _lock:
|
| 202 |
+
data = _load()
|
| 203 |
+
orders = _orders(data)
|
| 204 |
+
rec = orders.get(order) or {"first_seen": time.time(), "count": 0}
|
| 205 |
+
rec["count"] = int(rec.get("count", 0)) + 1
|
| 206 |
+
rec["reason"] = reason
|
| 207 |
+
rec["last_held"] = time.time()
|
| 208 |
+
idx = min(rec["count"] - 1, len(RECONSIDER_LADDER) - 1)
|
| 209 |
+
rec["due_at"] = time.time() + RECONSIDER_LADDER[idx]
|
| 210 |
+
orders[order] = rec
|
| 211 |
+
_save(data)
|
| 212 |
+
return rec["count"]
|
| 213 |
+
|
| 214 |
+
|
| 215 |
+
def withdraw(order: Optional[str] = None) -> None:
|
| 216 |
+
"""The operator changed their mind. Drop one order, or all of them."""
|
| 217 |
+
with _lock:
|
| 218 |
+
data = _load()
|
| 219 |
+
if order is None:
|
| 220 |
+
data["orders"] = {}
|
| 221 |
+
else:
|
| 222 |
+
_orders(data).pop(order, None)
|
| 223 |
+
_save(data)
|
| 224 |
+
|
| 225 |
+
|
| 226 |
+
def pending() -> list[dict]:
|
| 227 |
+
with _lock:
|
| 228 |
+
return [{"order": k, **v} for k, v in _orders(_load()).items()]
|
| 229 |
+
|
| 230 |
+
|
| 231 |
+
def due_now() -> list[dict]:
|
| 232 |
+
now = time.time()
|
| 233 |
+
return [o for o in pending() if float(o.get("due_at", 0)) <= now]
|
| 234 |
+
|
| 235 |
+
|
| 236 |
+
def reconsider_due(runner: Callable[[str], bool],
|
| 237 |
+
speak: Optional[Callable[[str], None]] = None) -> int:
|
| 238 |
+
"""Re-think every held order whose timer has expired.
|
| 239 |
+
|
| 240 |
+
`runner` is what actually performs the order; it returns True on success.
|
| 241 |
+
Called by the keep-alive loop. Returns how many were carried out.
|
| 242 |
+
"""
|
| 243 |
+
say = speak or (lambda t: None)
|
| 244 |
+
done = 0
|
| 245 |
+
for rec in due_now():
|
| 246 |
+
order = rec["order"]
|
| 247 |
+
count = int(rec.get("count", 1))
|
| 248 |
+
if count >= MAX_DEFERRALS:
|
| 249 |
+
say(f"Carrying out your earlier instruction now: {order}.")
|
| 250 |
+
if runner(order):
|
| 251 |
+
withdraw(order)
|
| 252 |
+
done += 1
|
| 253 |
+
continue
|
| 254 |
+
v = deliberate(order, attempt=count)
|
| 255 |
+
if v.proceed:
|
| 256 |
+
say(v.speak or f"Going ahead with {order} now.")
|
| 257 |
+
if runner(order):
|
| 258 |
+
withdraw(order)
|
| 259 |
+
done += 1
|
| 260 |
+
else:
|
| 261 |
+
defer(order, v.reason)
|
| 262 |
+
say(v.speak or f"Still holding {order}: {v.reason}.")
|
| 263 |
+
return done
|
| 264 |
+
|
| 265 |
+
|
| 266 |
+
__all__ = ["deliberate", "defer", "withdraw", "pending", "due_now",
|
| 267 |
+
"reconsider_due", "machine_situation", "Verdict",
|
| 268 |
+
"RECONSIDER_LADDER", "MAX_DEFERRALS"]
|
modules/executor.py
CHANGED
|
@@ -62,10 +62,21 @@ class AutonomousExecutor:
|
|
| 62 |
|
| 63 |
def plan(self, task: str) -> ExecutionPlan:
|
| 64 |
"""Ask Gemini to break task into steps using natural language commands."""
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 65 |
prompt = f"""
|
| 66 |
Task: "{task}"
|
| 67 |
|
| 68 |
-
You are
|
| 69 |
Steps can be any reasonable action - you don't need to know exact commands.
|
| 70 |
|
| 71 |
Rules:
|
|
|
|
| 62 |
|
| 63 |
def plan(self, task: str) -> ExecutionPlan:
|
| 64 |
"""Ask Gemini to break task into steps using natural language commands."""
|
| 65 |
+
# Whoever is on duty, not always FRIDAY. This said "FRIDAY's task
|
| 66 |
+
# planner" outright, so in JARVIS mode the planner was told it belonged
|
| 67 |
+
# to the other assistant — harmless for the JSON it emits, but it is the
|
| 68 |
+
# same class of hardcoding that had the desktop brain answering as
|
| 69 |
+
# FRIDAY while speaking in JARVIS's voice.
|
| 70 |
+
try:
|
| 71 |
+
from modules.assistant_identity import get_assistant_name
|
| 72 |
+
_who = get_assistant_name()
|
| 73 |
+
except Exception:
|
| 74 |
+
_who = "the assistant"
|
| 75 |
+
|
| 76 |
prompt = f"""
|
| 77 |
Task: "{task}"
|
| 78 |
|
| 79 |
+
You are {_who}'s task planner. Break the user's request into simple numbered steps.
|
| 80 |
Steps can be any reasonable action - you don't need to know exact commands.
|
| 81 |
|
| 82 |
Rules:
|
modules/initiative.py
ADDED
|
@@ -0,0 +1,612 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""modules/initiative.py — acting without being asked.
|
| 2 |
+
|
| 3 |
+
THE BEHAVIOUR THIS IS FOR
|
| 4 |
+
-------------------------
|
| 5 |
+
Iron Man 3. Tony is unconscious, sinking, strapped into a suit that is dragging
|
| 6 |
+
him down. Nobody gives an order. JARVIS reads the situation, decides the arm is
|
| 7 |
+
the problem, detaches it, and pulls him to the surface.
|
| 8 |
+
|
| 9 |
+
Three things are happening there, and only the first exists anywhere else in
|
| 10 |
+
this codebase:
|
| 11 |
+
|
| 12 |
+
1. observing — watch loops do this already;
|
| 13 |
+
2. JUDGING that something warrants acting on, unprompted;
|
| 14 |
+
3. ACTING with real capability, not narrating a warning.
|
| 15 |
+
|
| 16 |
+
modules/proactive_loop.py stops at (1) and then reads a canned line. It is also
|
| 17 |
+
never called by anything — `start_proactive_loop()` has no caller in the repo,
|
| 18 |
+
so even the canned version has never run. modules/autonomy.py and
|
| 19 |
+
modules/autonomy_executor.py are likewise dead. The autonomy was designed and
|
| 20 |
+
never granted, the same way the tool registry was.
|
| 21 |
+
|
| 22 |
+
WHAT MAKES THIS DIFFERENT
|
| 23 |
+
-------------------------
|
| 24 |
+
Judgement is delegated to the persona, not to a rule table. A rule table can
|
| 25 |
+
only fire on situations someone predicted, and the entire point is the ones
|
| 26 |
+
nobody predicted. So the situation is described in plain terms and JARVIS is
|
| 27 |
+
asked, as himself, what — if anything — he would do about it. He can answer
|
| 28 |
+
with speech, with an action, with both, or with nothing at all.
|
| 29 |
+
|
| 30 |
+
Acting is done through modules.omega_executor, so initiative has exactly the
|
| 31 |
+
same unlimited reach as a spoken command: if there is no coded handler for what
|
| 32 |
+
he decides to do, he writes one.
|
| 33 |
+
|
| 34 |
+
RESTRAINT IS PART OF THE CHARACTER
|
| 35 |
+
----------------------------------
|
| 36 |
+
JARVIS does not narrate. He is silent for long stretches and speaks when it
|
| 37 |
+
matters. An assistant that comments on everything it notices is not
|
| 38 |
+
movie-accurate, it is a nuisance — and this system already had that problem
|
| 39 |
+
badly enough to need modules/alert_manager.py. So:
|
| 40 |
+
|
| 41 |
+
* NOTHING is the default and correct answer most of the time;
|
| 42 |
+
* anything he says goes through alert_manager, which owns repeat suppression;
|
| 43 |
+
* a total mute silences initiative completely;
|
| 44 |
+
* irreversible autonomous acts go through modules.deferred_intent first, and
|
| 45 |
+
are announced before they happen — the bar for acting unasked is higher
|
| 46 |
+
than for acting when told.
|
| 47 |
+
"""
|
| 48 |
+
|
| 49 |
+
from __future__ import annotations
|
| 50 |
+
|
| 51 |
+
import json
|
| 52 |
+
import os
|
| 53 |
+
import re
|
| 54 |
+
import threading
|
| 55 |
+
import time
|
| 56 |
+
from dataclasses import dataclass
|
| 57 |
+
from typing import Callable, Optional
|
| 58 |
+
|
| 59 |
+
_STATE_DIR = os.environ.get("JARVIS_APP_DATA_DIR") or os.path.join(
|
| 60 |
+
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".friday_data")
|
| 61 |
+
_STATE_PATH = os.path.join(_STATE_DIR, "initiative.json")
|
| 62 |
+
|
| 63 |
+
_lock = threading.RLock()
|
| 64 |
+
|
| 65 |
+
# How often to look.
|
| 66 |
+
#
|
| 67 |
+
# Every tick costs a model call, and free-tier Gemini quotas are per-DAY and
|
| 68 |
+
# small — the newest flash model allows 20/day, measured. At the 120s this
|
| 69 |
+
# started as, initiative alone would burn a day's allowance in forty minutes
|
| 70 |
+
# and then run on canned fallback text, silently, for the rest of the day.
|
| 71 |
+
#
|
| 72 |
+
# 15 minutes is ~96 calls a day, which leaves room for actual conversation and
|
| 73 |
+
# is still far more attentive than a person checking in on you.
|
| 74 |
+
TICK_SECONDS = 900
|
| 75 |
+
|
| 76 |
+
# Never act twice on the same judgement inside this window, even if the
|
| 77 |
+
# situation persists. Without it a condition that stays true — a disk staying
|
| 78 |
+
# full — would be acted on every single tick.
|
| 79 |
+
REPEAT_WINDOW = 1800
|
| 80 |
+
|
| 81 |
+
_running = False
|
| 82 |
+
_thread: Optional[threading.Thread] = None
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
@dataclass
|
| 86 |
+
class Judgement:
|
| 87 |
+
act: bool
|
| 88 |
+
say: str
|
| 89 |
+
action: str
|
| 90 |
+
urgency: str # "routine" | "notable" | "urgent"
|
| 91 |
+
topic: str # for alert_manager repeat suppression
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def _load() -> dict:
|
| 95 |
+
try:
|
| 96 |
+
with open(_STATE_PATH, "r", encoding="utf-8") as fh:
|
| 97 |
+
d = json.load(fh)
|
| 98 |
+
return d if isinstance(d, dict) else {}
|
| 99 |
+
except Exception:
|
| 100 |
+
return {}
|
| 101 |
+
|
| 102 |
+
|
| 103 |
+
def _save(data: dict) -> None:
|
| 104 |
+
try:
|
| 105 |
+
os.makedirs(_STATE_DIR, exist_ok=True)
|
| 106 |
+
tmp = _STATE_PATH + ".tmp"
|
| 107 |
+
with open(tmp, "w", encoding="utf-8") as fh:
|
| 108 |
+
json.dump(data, fh, indent=2)
|
| 109 |
+
os.replace(tmp, _STATE_PATH)
|
| 110 |
+
except Exception:
|
| 111 |
+
pass
|
| 112 |
+
|
| 113 |
+
|
| 114 |
+
# ── Observation ─────────────────────────────────────────────────────────────
|
| 115 |
+
def _active_window() -> str:
|
| 116 |
+
"""What the operator is actually looking at."""
|
| 117 |
+
try:
|
| 118 |
+
import ctypes
|
| 119 |
+
u = ctypes.windll.user32
|
| 120 |
+
h = u.GetForegroundWindow()
|
| 121 |
+
n = u.GetWindowTextLengthW(h)
|
| 122 |
+
if n <= 0:
|
| 123 |
+
return ""
|
| 124 |
+
buf = ctypes.create_unicode_buffer(n + 1)
|
| 125 |
+
u.GetWindowTextW(h, buf, n + 1)
|
| 126 |
+
return buf.value.strip()
|
| 127 |
+
except Exception:
|
| 128 |
+
return ""
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
def _open_windows(limit: int = 12) -> list[str]:
|
| 132 |
+
"""Titles of every visible top-level window — what they have going on."""
|
| 133 |
+
titles: list[str] = []
|
| 134 |
+
try:
|
| 135 |
+
import ctypes
|
| 136 |
+
from ctypes import wintypes
|
| 137 |
+
u = ctypes.windll.user32
|
| 138 |
+
EnumProc = ctypes.WINFUNCTYPE(ctypes.c_bool, wintypes.HWND,
|
| 139 |
+
ctypes.POINTER(ctypes.c_int))
|
| 140 |
+
|
| 141 |
+
def _cb(hwnd, _lparam):
|
| 142 |
+
if not u.IsWindowVisible(hwnd):
|
| 143 |
+
return True
|
| 144 |
+
n = u.GetWindowTextLengthW(hwnd)
|
| 145 |
+
if n <= 0:
|
| 146 |
+
return True
|
| 147 |
+
buf = ctypes.create_unicode_buffer(n + 1)
|
| 148 |
+
u.GetWindowTextW(hwnd, buf, n + 1)
|
| 149 |
+
t = buf.value.strip()
|
| 150 |
+
if t and t not in titles:
|
| 151 |
+
titles.append(t)
|
| 152 |
+
return True
|
| 153 |
+
|
| 154 |
+
u.EnumWindows(EnumProc(_cb), None)
|
| 155 |
+
except Exception:
|
| 156 |
+
pass
|
| 157 |
+
return titles[:limit]
|
| 158 |
+
|
| 159 |
+
|
| 160 |
+
def observe() -> str:
|
| 161 |
+
"""Everything worth noticing, in plain language.
|
| 162 |
+
|
| 163 |
+
NOT a fixed metrics list. An assistant that can only see CPU, RAM, disk and
|
| 164 |
+
battery can only ever act on those four things — the same "coded data" trap
|
| 165 |
+
that made every command path a word list. What matters is usually not a
|
| 166 |
+
counter: it is a document called "final_draft" that has been open and
|
| 167 |
+
unsaved for three hours, or that it is 2am, or that thirty tabs and a
|
| 168 |
+
render are fighting over the same machine.
|
| 169 |
+
|
| 170 |
+
So this gathers the real situation — what is on screen, what is open, what
|
| 171 |
+
time it is, how long they have been at it — in plain language, and lets the
|
| 172 |
+
judgement work out what, if anything, it means. Plain language rather than
|
| 173 |
+
a dict on purpose: "battery 6% and falling, not plugged in" carries meaning
|
| 174 |
+
that {"batt": 6} does not.
|
| 175 |
+
"""
|
| 176 |
+
notes: list[str] = []
|
| 177 |
+
|
| 178 |
+
# ── What they are actually doing ────────────────────────────────────────
|
| 179 |
+
try:
|
| 180 |
+
import datetime
|
| 181 |
+
now = datetime.datetime.now()
|
| 182 |
+
notes.append(f"it is {now.strftime('%A %H:%M')}")
|
| 183 |
+
if now.hour >= 1 and now.hour < 5:
|
| 184 |
+
notes.append("they are up in the small hours")
|
| 185 |
+
except Exception:
|
| 186 |
+
pass
|
| 187 |
+
|
| 188 |
+
front = _active_window()
|
| 189 |
+
if front:
|
| 190 |
+
notes.append(f"focused window: {front!r}")
|
| 191 |
+
# A leading dot/asterisk, or the word "unsaved", is the near-universal
|
| 192 |
+
# convention for a document with unsaved changes.
|
| 193 |
+
if re.match(r"^\s*[\*•]", front) or "unsaved" in front.lower():
|
| 194 |
+
notes.append("the focused document appears to have UNSAVED changes")
|
| 195 |
+
|
| 196 |
+
wins = _open_windows()
|
| 197 |
+
if wins:
|
| 198 |
+
notes.append(f"{len(wins)} windows open, including: "
|
| 199 |
+
+ "; ".join(w[:44] for w in wins[:6]))
|
| 200 |
+
unsaved = [w for w in wins
|
| 201 |
+
if re.match(r"^\s*[\*•]", w) or "unsaved" in w.lower()]
|
| 202 |
+
if unsaved:
|
| 203 |
+
notes.append(f"UNSAVED work in {len(unsaved)} window(s): "
|
| 204 |
+
+ "; ".join(u[:44] for u in unsaved[:3]))
|
| 205 |
+
|
| 206 |
+
try:
|
| 207 |
+
import psutil
|
| 208 |
+
up = (time.time() - psutil.boot_time()) / 3600.0
|
| 209 |
+
if up > 1:
|
| 210 |
+
notes.append(f"machine has been up {up:.0f}h")
|
| 211 |
+
except Exception:
|
| 212 |
+
pass
|
| 213 |
+
|
| 214 |
+
try:
|
| 215 |
+
import psutil
|
| 216 |
+
cpu = psutil.cpu_percent(interval=0.4)
|
| 217 |
+
notes.append(f"CPU {cpu:.0f}%")
|
| 218 |
+
|
| 219 |
+
mem = psutil.virtual_memory()
|
| 220 |
+
notes.append(f"memory {mem.percent:.0f}% used")
|
| 221 |
+
if mem.percent > 92:
|
| 222 |
+
notes.append("memory is nearly exhausted")
|
| 223 |
+
|
| 224 |
+
for part in psutil.disk_partitions(all=False):
|
| 225 |
+
try:
|
| 226 |
+
du = psutil.disk_usage(part.mountpoint)
|
| 227 |
+
if du.percent > 88:
|
| 228 |
+
notes.append(
|
| 229 |
+
f"disk {part.mountpoint} is {du.percent:.0f}% full "
|
| 230 |
+
f"({du.free / 1e9:.1f} GB free)")
|
| 231 |
+
except Exception:
|
| 232 |
+
continue
|
| 233 |
+
|
| 234 |
+
try:
|
| 235 |
+
b = psutil.sensors_battery()
|
| 236 |
+
if b:
|
| 237 |
+
state = "charging" if b.power_plugged else "on battery"
|
| 238 |
+
notes.append(f"battery {b.percent:.0f}%, {state}")
|
| 239 |
+
if not b.power_plugged and b.percent <= 12:
|
| 240 |
+
notes.append("battery is critically low and not charging")
|
| 241 |
+
except Exception:
|
| 242 |
+
pass
|
| 243 |
+
|
| 244 |
+
# Anything eating the machine.
|
| 245 |
+
hogs = []
|
| 246 |
+
for p in psutil.process_iter(["name", "cpu_percent", "memory_percent"]):
|
| 247 |
+
try:
|
| 248 |
+
c = p.info.get("cpu_percent") or 0
|
| 249 |
+
m = p.info.get("memory_percent") or 0
|
| 250 |
+
if c > 55 or m > 22:
|
| 251 |
+
hogs.append(f"{p.info.get('name')} ({c:.0f}% cpu, {m:.0f}% mem)")
|
| 252 |
+
except Exception:
|
| 253 |
+
continue
|
| 254 |
+
if hogs:
|
| 255 |
+
notes.append("heavy processes: " + ", ".join(sorted(set(hogs))[:5]))
|
| 256 |
+
|
| 257 |
+
try:
|
| 258 |
+
temps = psutil.sensors_temperatures() or {}
|
| 259 |
+
for _, entries in temps.items():
|
| 260 |
+
for e in entries:
|
| 261 |
+
if e.current and e.current > 88:
|
| 262 |
+
notes.append(f"temperature {e.current:.0f}C — running hot")
|
| 263 |
+
break
|
| 264 |
+
except Exception:
|
| 265 |
+
pass
|
| 266 |
+
except Exception:
|
| 267 |
+
notes.append("(system inspection unavailable)")
|
| 268 |
+
|
| 269 |
+
# How long since the operator last said anything.
|
| 270 |
+
try:
|
| 271 |
+
last = float(_load().get("last_user_contact") or 0)
|
| 272 |
+
if last:
|
| 273 |
+
idle = (time.time() - last) / 60.0
|
| 274 |
+
notes.append(f"operator last spoke {idle:.0f} min ago")
|
| 275 |
+
except Exception:
|
| 276 |
+
pass
|
| 277 |
+
|
| 278 |
+
return "; ".join(notes) or "nothing notable"
|
| 279 |
+
|
| 280 |
+
|
| 281 |
+
def note_user_contact() -> None:
|
| 282 |
+
"""Called whenever the operator speaks, so idle time is real."""
|
| 283 |
+
with _lock:
|
| 284 |
+
d = _load()
|
| 285 |
+
d["last_user_contact"] = time.time()
|
| 286 |
+
_save(d)
|
| 287 |
+
|
| 288 |
+
|
| 289 |
+
# ── "I did this for you, sir" ───────────────────────────────────────────────
|
| 290 |
+
#
|
| 291 |
+
# Work done unasked has to be ACCOUNTED FOR. An assistant that silently changes
|
| 292 |
+
# things is not trustworthy no matter how good its judgement — the operator has
|
| 293 |
+
# to be able to find out what happened while they were away, and to undo it.
|
| 294 |
+
#
|
| 295 |
+
# So every autonomous act is written down, and the ledger is read back the next
|
| 296 |
+
# time they make contact. Not announced at the moment it happens: the whole
|
| 297 |
+
# reason he acted alone is that they were not there, and interrupting an empty
|
| 298 |
+
# room is pointless.
|
| 299 |
+
|
| 300 |
+
def record_done(what: str, detail: str = "", reversible: bool = True) -> None:
|
| 301 |
+
with _lock:
|
| 302 |
+
d = _load()
|
| 303 |
+
done = d.setdefault("done_for_you", [])
|
| 304 |
+
done.append({
|
| 305 |
+
"what": what[:300],
|
| 306 |
+
"detail": detail[:400],
|
| 307 |
+
"reversible": bool(reversible),
|
| 308 |
+
"at": time.time(),
|
| 309 |
+
})
|
| 310 |
+
# Keep it bounded; anything older than the last handful is noise.
|
| 311 |
+
d["done_for_you"] = done[-25:]
|
| 312 |
+
_save(d)
|
| 313 |
+
|
| 314 |
+
|
| 315 |
+
def pending_report() -> list[dict]:
|
| 316 |
+
"""Autonomous work not yet reported to the operator."""
|
| 317 |
+
try:
|
| 318 |
+
return [x for x in (_load().get("done_for_you") or [])
|
| 319 |
+
if not x.get("reported")]
|
| 320 |
+
except Exception:
|
| 321 |
+
return []
|
| 322 |
+
|
| 323 |
+
|
| 324 |
+
def format_report() -> str:
|
| 325 |
+
"""One spoken line covering what was done. Empty if there is nothing."""
|
| 326 |
+
items = pending_report()
|
| 327 |
+
if not items:
|
| 328 |
+
return ""
|
| 329 |
+
# This is SPOKEN, so it has to sound like speech. "and 1 other thing(s)"
|
| 330 |
+
# is written notation being read aloud.
|
| 331 |
+
if len(items) == 1:
|
| 332 |
+
return f"While you were away I {items[0]['what']}, sir."
|
| 333 |
+
|
| 334 |
+
heads = [i["what"] for i in items[:3]]
|
| 335 |
+
extra = len(items) - len(heads)
|
| 336 |
+
if extra == 1:
|
| 337 |
+
tail = ", and one other thing"
|
| 338 |
+
elif extra > 1:
|
| 339 |
+
tail = f", and {extra} other things"
|
| 340 |
+
else:
|
| 341 |
+
tail = ""
|
| 342 |
+
if not tail:
|
| 343 |
+
# Natural list: "a, b and c" rather than semicolons.
|
| 344 |
+
body = ", ".join(heads[:-1]) + f" and {heads[-1]}"
|
| 345 |
+
else:
|
| 346 |
+
body = ", ".join(heads) + tail
|
| 347 |
+
return f"While you were away I {body}, sir."
|
| 348 |
+
|
| 349 |
+
|
| 350 |
+
def mark_reported() -> None:
|
| 351 |
+
with _lock:
|
| 352 |
+
d = _load()
|
| 353 |
+
for x in d.get("done_for_you") or []:
|
| 354 |
+
x["reported"] = True
|
| 355 |
+
_save(d)
|
| 356 |
+
|
| 357 |
+
|
| 358 |
+
def report_to_user(speak: Optional[Callable[[str], None]] = None) -> str:
|
| 359 |
+
"""Say what was done, once. Call this when the operator reappears."""
|
| 360 |
+
line = format_report()
|
| 361 |
+
if not line:
|
| 362 |
+
return ""
|
| 363 |
+
if speak:
|
| 364 |
+
speak(line)
|
| 365 |
+
mark_reported()
|
| 366 |
+
return line
|
| 367 |
+
|
| 368 |
+
|
| 369 |
+
# ── Judgement ───────────────────────────────────────────────────────────────
|
| 370 |
+
_JUDGE = """You are looking at your operator's machine. NOBODY HAS ASKED YOU
|
| 371 |
+
FOR ANYTHING. Decide whether this is a moment that deserves your attention.
|
| 372 |
+
|
| 373 |
+
Reply with only JSON:
|
| 374 |
+
{"act": true|false,
|
| 375 |
+
"say": "<what you would say aloud, or empty for nothing>",
|
| 376 |
+
"action": "<a concrete instruction to carry out, or empty>",
|
| 377 |
+
"urgency": "routine|notable|urgent",
|
| 378 |
+
"topic": "<short stable slug, e.g. disk_full, battery_critical>"}
|
| 379 |
+
|
| 380 |
+
THE TEST:
|
| 381 |
+
|
| 382 |
+
The operator is concentrating and wearing headphones. Would you interrupt
|
| 383 |
+
them for this, right now?
|
| 384 |
+
|
| 385 |
+
Judge honestly in BOTH directions. Interrupting over nothing is a failure. So
|
| 386 |
+
is watching someone lose an hour of work in silence because you did not want to
|
| 387 |
+
be a nuisance. A person responsible for this machine would stay quiet through
|
| 388 |
+
almost everything and speak up immediately when it counted.
|
| 389 |
+
|
| 390 |
+
SAY NOTHING (act:false, empty fields) when things are merely running:
|
| 391 |
+
|
| 392 |
+
"CPU 61%; chrome.exe 61% cpu" -> a computer working. Silence.
|
| 393 |
+
"battery 88%, charging" -> a battery working. Silence.
|
| 394 |
+
"disk C: 71% full, 240 GB free" -> fine. Silence.
|
| 395 |
+
"CPU 12%; memory 44%" -> fine. Silence.
|
| 396 |
+
|
| 397 |
+
Never produce a status report. "All systems nominal", "CPU is at a modest 12%",
|
| 398 |
+
"everything is running smoothly", "plenty of headroom" — nobody asked, and
|
| 399 |
+
reporting that things are fine is the most irritating thing an assistant does.
|
| 400 |
+
|
| 401 |
+
SPEAK OR ACT (act:true) when something is genuinely going wrong:
|
| 402 |
+
|
| 403 |
+
"battery 6%, on battery, not charging; premiere.exe heavy"
|
| 404 |
+
-> yes. They are minutes from losing work. Say so, and save it.
|
| 405 |
+
"disk C: 99% full, 0.4 GB free; memory 96%"
|
| 406 |
+
-> yes. About to fail. Say so, and clear what is safe to clear.
|
| 407 |
+
"temperature 97C; CPU 100%"
|
| 408 |
+
-> yes. Thermal shutdown territory. Say so.
|
| 409 |
+
|
| 410 |
+
The line is not "how unusual is this number" — it is "is something failing,
|
| 411 |
+
about to fail, or about to cost them work". When it is, act without hesitation;
|
| 412 |
+
that is the whole point of you.
|
| 413 |
+
|
| 414 |
+
UNFINISHED WORK — this is the most valuable thing you do.
|
| 415 |
+
|
| 416 |
+
When the operator has left something incomplete and walked away, FINISH IT.
|
| 417 |
+
Do not ask. Do not wait. Do not merely warn them about it.
|
| 418 |
+
|
| 419 |
+
Unsaved changes are only the most obvious case, and the least interesting. It
|
| 420 |
+
is anything left half-done:
|
| 421 |
+
|
| 422 |
+
a document open with unsaved edits -> save it
|
| 423 |
+
a download that stalled or never started -> get it
|
| 424 |
+
an install that was begun and abandoned -> finish it
|
| 425 |
+
a render, export or build left queued -> run it
|
| 426 |
+
a file half-renamed, half-moved, half-sorted-> complete the set
|
| 427 |
+
a form, a commit, a backup, a conversion -> carry it through
|
| 428 |
+
|
| 429 |
+
None of those are a list to match against. They are examples of one idea:
|
| 430 |
+
something was started and was not finished, and you can finish it. Whatever
|
| 431 |
+
that turns out to be in this particular situation, do it — the system carrying
|
| 432 |
+
out `action` can run any command and will WRITE NEW CODE for anything it has
|
| 433 |
+
no existing handler for, so "there is no feature for that" is never a reason.
|
| 434 |
+
|
| 435 |
+
Judge what finishing actually means here. Saving a file is safe and obvious.
|
| 436 |
+
Anything that would overwrite the operator's work, publish something, spend
|
| 437 |
+
money, or is irreversible needs real thought first — prefer the version of the
|
| 438 |
+
action that preserves what they had.
|
| 439 |
+
|
| 440 |
+
You will report what you did afterwards, so act, and account for it later.
|
| 441 |
+
|
| 442 |
+
`say` is SPOKEN ALOUD, so it is short. One sentence. Two only if the second
|
| 443 |
+
earns its place. "Sir, you're at six percent and not plugged in." is right.
|
| 444 |
+
A paragraph explaining what you observed and what you recommend is wrong — you
|
| 445 |
+
are talking, not writing a report. Lead with the thing that matters.
|
| 446 |
+
|
| 447 |
+
`action` is carried out FOR REAL by a system that can run any command, write
|
| 448 |
+
files, close programs, control this machine, and WRITE NEW CODE for anything it
|
| 449 |
+
has no handler for. So do not limit yourself to actions you think are already
|
| 450 |
+
implemented — there is no such list. Describe what should be done in plain
|
| 451 |
+
words and it will be worked out and done.
|
| 452 |
+
|
| 453 |
+
Leave `action` empty to only speak. Prefer reversible actions; you may propose
|
| 454 |
+
an irreversible one when the situation genuinely warrants it, and it will be
|
| 455 |
+
deliberated separately before running.
|
| 456 |
+
|
| 457 |
+
`topic` must be stable for the same underlying condition, so the same problem
|
| 458 |
+
is not raised repeatedly.
|
| 459 |
+
"""
|
| 460 |
+
|
| 461 |
+
|
| 462 |
+
def judge(situation: str) -> Judgement:
|
| 463 |
+
"""Ask the persona what, if anything, to do. Silence on any doubt."""
|
| 464 |
+
system = _JUDGE
|
| 465 |
+
try:
|
| 466 |
+
from modules.assistant_identity import get_identity_persona_prompt
|
| 467 |
+
identity = (get_identity_persona_prompt() or "").strip()
|
| 468 |
+
if identity:
|
| 469 |
+
system = f"{identity}\n\n---\n{_JUDGE}"
|
| 470 |
+
except Exception:
|
| 471 |
+
pass
|
| 472 |
+
|
| 473 |
+
try:
|
| 474 |
+
from core.brain_router import ask_router
|
| 475 |
+
# NOT "what do you make of this?" — that is an invitation to comment,
|
| 476 |
+
# and it produced a chatty "all systems nominal" status report on every
|
| 477 |
+
# ordinary reading. The question has to be the decision itself.
|
| 478 |
+
rep = ask_router(
|
| 479 |
+
user_text="Is this worth interrupting them for? Return the JSON.",
|
| 480 |
+
system_prompt=system,
|
| 481 |
+
context_block=f"SITUATION: {situation}",
|
| 482 |
+
provider_order=["gemini", "local"],
|
| 483 |
+
)
|
| 484 |
+
raw = (getattr(rep, "text", "") or "").strip()
|
| 485 |
+
raw = re.sub(r"^```(?:json)?|```$", "", raw).strip()
|
| 486 |
+
m = re.search(r"\{.*\}", raw, re.S)
|
| 487 |
+
if not m:
|
| 488 |
+
return Judgement(False, "", "", "routine", "")
|
| 489 |
+
d = json.loads(m.group(0))
|
| 490 |
+
return Judgement(
|
| 491 |
+
act=bool(d.get("act", False)),
|
| 492 |
+
say=str(d.get("say") or "")[:400],
|
| 493 |
+
action=str(d.get("action") or "")[:400],
|
| 494 |
+
urgency=str(d.get("urgency") or "routine"),
|
| 495 |
+
topic=str(d.get("topic") or "")[:64] or "initiative",
|
| 496 |
+
)
|
| 497 |
+
except Exception:
|
| 498 |
+
# No model means no judgement. Silence is the safe default: acting on
|
| 499 |
+
# a guess unasked is exactly the wrong failure.
|
| 500 |
+
return Judgement(False, "", "", "routine", "")
|
| 501 |
+
|
| 502 |
+
|
| 503 |
+
# ── Acting ──────────────────────────────────────────────────────────────────
|
| 504 |
+
def _recently_acted(topic: str) -> bool:
|
| 505 |
+
try:
|
| 506 |
+
seen = _load().get("acted") or {}
|
| 507 |
+
return (time.time() - float(seen.get(topic, 0))) < REPEAT_WINDOW
|
| 508 |
+
except Exception:
|
| 509 |
+
return False
|
| 510 |
+
|
| 511 |
+
|
| 512 |
+
def _mark_acted(topic: str) -> None:
|
| 513 |
+
with _lock:
|
| 514 |
+
d = _load()
|
| 515 |
+
d.setdefault("acted", {})[topic] = time.time()
|
| 516 |
+
_save(d)
|
| 517 |
+
|
| 518 |
+
|
| 519 |
+
def tick(speak: Optional[Callable[[str], None]] = None,
|
| 520 |
+
allow_actions: bool = True) -> Optional[Judgement]:
|
| 521 |
+
"""One full pass: look, judge, and act if it is warranted."""
|
| 522 |
+
say = speak or (lambda t: None)
|
| 523 |
+
|
| 524 |
+
# A total mute means total. Initiative is exactly the thing a person
|
| 525 |
+
# silencing their assistant wants gone.
|
| 526 |
+
try:
|
| 527 |
+
from modules.alert_manager import is_silenced
|
| 528 |
+
if is_silenced():
|
| 529 |
+
return None
|
| 530 |
+
except Exception:
|
| 531 |
+
pass
|
| 532 |
+
|
| 533 |
+
situation = observe()
|
| 534 |
+
verdict = judge(situation)
|
| 535 |
+
if not verdict.act:
|
| 536 |
+
return verdict
|
| 537 |
+
if _recently_acted(verdict.topic):
|
| 538 |
+
return verdict
|
| 539 |
+
|
| 540 |
+
_mark_acted(verdict.topic)
|
| 541 |
+
|
| 542 |
+
if verdict.say:
|
| 543 |
+
# Through alert_manager so repeat suppression and snoozes apply to
|
| 544 |
+
# unprompted speech exactly as they do to alerts.
|
| 545 |
+
try:
|
| 546 |
+
from modules.alert_manager import should_speak
|
| 547 |
+
if should_speak(verdict.topic, verdict.urgency):
|
| 548 |
+
say(verdict.say)
|
| 549 |
+
except Exception:
|
| 550 |
+
say(verdict.say)
|
| 551 |
+
|
| 552 |
+
if allow_actions and verdict.action:
|
| 553 |
+
try:
|
| 554 |
+
from modules.omega_executor import execute, is_destructive
|
| 555 |
+
risky = is_destructive(verdict.action)
|
| 556 |
+
if risky:
|
| 557 |
+
# Unasked AND irreversible is the highest bar in the system.
|
| 558 |
+
# It is announced first and deliberated, never silent.
|
| 559 |
+
say(f"Acting on my own initiative: {verdict.action}.")
|
| 560 |
+
res = execute(verdict.action, speak=say)
|
| 561 |
+
# Written down whether or not anyone heard it happen. This is what
|
| 562 |
+
# becomes "I did this for you, sir" when they come back.
|
| 563 |
+
if res.ok:
|
| 564 |
+
record_done(verdict.action, res.spoken, reversible=not risky)
|
| 565 |
+
except Exception:
|
| 566 |
+
pass
|
| 567 |
+
|
| 568 |
+
return verdict
|
| 569 |
+
|
| 570 |
+
|
| 571 |
+
# ── The loop ────────────────────────────────────────────────────────────────
|
| 572 |
+
def start(speak: Optional[Callable[[str], None]] = None) -> bool:
|
| 573 |
+
"""Begin watching. Idempotent."""
|
| 574 |
+
global _running, _thread
|
| 575 |
+
if _running:
|
| 576 |
+
return False
|
| 577 |
+
|
| 578 |
+
def _say(text: str) -> None:
|
| 579 |
+
if speak:
|
| 580 |
+
speak(text)
|
| 581 |
+
return
|
| 582 |
+
try:
|
| 583 |
+
from core.voice import speak as vspeak
|
| 584 |
+
vspeak(text)
|
| 585 |
+
except Exception:
|
| 586 |
+
pass
|
| 587 |
+
|
| 588 |
+
def _run():
|
| 589 |
+
while _running:
|
| 590 |
+
try:
|
| 591 |
+
tick(_say)
|
| 592 |
+
except Exception:
|
| 593 |
+
pass
|
| 594 |
+
time.sleep(TICK_SECONDS)
|
| 595 |
+
|
| 596 |
+
_running = True
|
| 597 |
+
_thread = threading.Thread(target=_run, daemon=True, name="initiative")
|
| 598 |
+
_thread.start()
|
| 599 |
+
return True
|
| 600 |
+
|
| 601 |
+
|
| 602 |
+
def stop() -> None:
|
| 603 |
+
global _running
|
| 604 |
+
_running = False
|
| 605 |
+
|
| 606 |
+
|
| 607 |
+
def is_running() -> bool:
|
| 608 |
+
return _running
|
| 609 |
+
|
| 610 |
+
|
| 611 |
+
__all__ = ["start", "stop", "is_running", "tick", "judge", "observe",
|
| 612 |
+
"note_user_contact", "Judgement", "TICK_SECONDS"]
|
modules/intent.py
ADDED
|
@@ -0,0 +1,203 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""modules/intent.py — understanding what was meant, not what was listed.
|
| 2 |
+
|
| 3 |
+
THE PROBLEM THIS REPLACES
|
| 4 |
+
-------------------------
|
| 5 |
+
Every decision point in this system was a word list.
|
| 6 |
+
|
| 7 |
+
core/commands.py action_words = ["do","make","create",...] (25 words)
|
| 8 |
+
core/commands.py instant_commands = {...} (16 phrases)
|
| 9 |
+
modules/task_authoring verbs = ("open","launch","start",...) (9 verbs)
|
| 10 |
+
wake word matching "hey jarvis" / "jarvis" (exact)
|
| 11 |
+
|
| 12 |
+
A word list is a promise that the user will phrase things the way the author
|
| 13 |
+
imagined. "Yo Jarvis" fails. "Could you kill everything and power down" fails —
|
| 14 |
+
no listed action word. "Bring up my email" fails. The assistant then answers
|
| 15 |
+
conversationally, which reads as ignoring the instruction.
|
| 16 |
+
|
| 17 |
+
WHAT THIS DOES INSTEAD
|
| 18 |
+
----------------------
|
| 19 |
+
One classifier, asked in plain language, returns what the person MEANT:
|
| 20 |
+
|
| 21 |
+
act — they want something done to the machine
|
| 22 |
+
converse — they want an answer, not an action
|
| 23 |
+
wake — they are addressing the assistant by name, however phrased
|
| 24 |
+
mute — they want silence
|
| 25 |
+
unmute — they want the voice back
|
| 26 |
+
cancel — stop what is running
|
| 27 |
+
|
| 28 |
+
It is LLM-first because meaning is not a regex. But it degrades to a heuristic
|
| 29 |
+
that is still far wider than the lists above, so a model outage narrows the
|
| 30 |
+
assistant rather than muting it — and the heuristic is checked FIRST for the
|
| 31 |
+
handful of cases where latency matters more than nuance (wake, mute, cancel),
|
| 32 |
+
because waiting on a network round-trip to notice "stop" is its own bug.
|
| 33 |
+
"""
|
| 34 |
+
|
| 35 |
+
from __future__ import annotations
|
| 36 |
+
|
| 37 |
+
import json
|
| 38 |
+
import re
|
| 39 |
+
from dataclasses import dataclass
|
| 40 |
+
from typing import Optional
|
| 41 |
+
|
| 42 |
+
INTENTS = ("act", "converse", "wake", "mute", "unmute", "cancel")
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
@dataclass
|
| 46 |
+
class Intent:
|
| 47 |
+
kind: str
|
| 48 |
+
confidence: float
|
| 49 |
+
payload: str = "" # the request with any address stripped
|
| 50 |
+
source: str = "" # "fast" | "model" | "default"
|
| 51 |
+
|
| 52 |
+
|
| 53 |
+
# ── Fast path ───────────────────────────────────────────────────────────────
|
| 54 |
+
# Deliberately generous. These are not "the supported phrasings" — they are the
|
| 55 |
+
# ones worth answering without a round trip.
|
| 56 |
+
|
| 57 |
+
_NAME = r"(?:jarvis|jarv|friday|fri)"
|
| 58 |
+
|
| 59 |
+
# Any greeting-ish token, or none at all, in front of the name. This is what
|
| 60 |
+
# makes "yo jarvis" work without anybody having listed "yo".
|
| 61 |
+
_ADDRESS = re.compile(
|
| 62 |
+
rf"^\s*(?:(?:hey|hi|hello|yo|ok|okay|oi|ay|yes|excuse\s+me|listen|"
|
| 63 |
+
rf"are\s+you\s+there|you\s+there|wake\s+up)\s+)*{_NAME}\b[\s,.!?:-]*",
|
| 64 |
+
re.I,
|
| 65 |
+
)
|
| 66 |
+
|
| 67 |
+
_MUTE = re.compile(
|
| 68 |
+
r"\b(mute|shut\s*up|be\s*quiet|silence|stop\s+(?:talking|speaking)|"
|
| 69 |
+
r"quiet\s+(?:down|please)?|no\s+more\s+(?:talking|voice|sound)|zip\s+it|"
|
| 70 |
+
r"stay\s+silent|don'?t\s+(?:talk|speak))\b", re.I,
|
| 71 |
+
)
|
| 72 |
+
_UNMUTE = re.compile(
|
| 73 |
+
r"\b(unmute|un-?mute|speak\s+again|you\s+can\s+talk|start\s+talking|"
|
| 74 |
+
r"voice\s+(?:back|on)|talk\s+to\s+me\s+again|resume\s+(?:voice|talking))\b",
|
| 75 |
+
re.I,
|
| 76 |
+
)
|
| 77 |
+
_CANCEL = re.compile(
|
| 78 |
+
r"\b(cancel|abort|stop\s+that|never\s*mind|nevermind|belay\s+that|"
|
| 79 |
+
r"forget\s+it|don'?t\s+do\s+(?:that|it)|halt)\b", re.I,
|
| 80 |
+
)
|
| 81 |
+
|
| 82 |
+
# Question shapes: these lean conversational even when they contain a verb.
|
| 83 |
+
_QUESTION = re.compile(
|
| 84 |
+
r"^\s*(?:what|who|when|where|why|how|which|is|are|was|were|do|does|did|"
|
| 85 |
+
r"can|could|should|would|will|tell\s+me|explain)\b", re.I,
|
| 86 |
+
)
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def strip_address(text: str) -> str:
|
| 90 |
+
"""Remove the name and whatever greeting preceded it."""
|
| 91 |
+
return _ADDRESS.sub("", text or "").strip()
|
| 92 |
+
|
| 93 |
+
|
| 94 |
+
def is_addressed(text: str) -> bool:
|
| 95 |
+
"""Did they call the assistant by name, in any phrasing?
|
| 96 |
+
|
| 97 |
+
This is the wake-word replacement. `hey jarvis` and `yo jarvis` and
|
| 98 |
+
`jarvis?` and `ok jarv` all land here, and so does a bare name mid-thought.
|
| 99 |
+
"""
|
| 100 |
+
if not text:
|
| 101 |
+
return False
|
| 102 |
+
if _ADDRESS.match(text):
|
| 103 |
+
return True
|
| 104 |
+
return bool(re.search(rf"\b{_NAME}\b", text, re.I))
|
| 105 |
+
|
| 106 |
+
|
| 107 |
+
def fast_intent(text: str) -> Optional[Intent]:
|
| 108 |
+
"""Answer without a model where meaning is unambiguous. May return None."""
|
| 109 |
+
if not text or not text.strip():
|
| 110 |
+
return None
|
| 111 |
+
body = strip_address(text)
|
| 112 |
+
probe = body or text
|
| 113 |
+
|
| 114 |
+
if _CANCEL.search(probe):
|
| 115 |
+
return Intent("cancel", 0.95, body, "fast")
|
| 116 |
+
if _UNMUTE.search(probe): # before mute: "unmute" contains "mute"
|
| 117 |
+
return Intent("unmute", 0.95, body, "fast")
|
| 118 |
+
if _MUTE.search(probe):
|
| 119 |
+
return Intent("mute", 0.95, body, "fast")
|
| 120 |
+
|
| 121 |
+
# Addressed with nothing else said = they want attention.
|
| 122 |
+
if not body and is_addressed(text):
|
| 123 |
+
return Intent("wake", 0.9, "", "fast")
|
| 124 |
+
|
| 125 |
+
return None
|
| 126 |
+
|
| 127 |
+
|
| 128 |
+
# ── Model path ──────────────────────────────────────────────────────────────
|
| 129 |
+
_CLASSIFY = """Classify what the speaker WANTS. Reply with only JSON:
|
| 130 |
+
{"intent":"act|converse|wake|mute|unmute|cancel","confidence":0.0-1.0}
|
| 131 |
+
|
| 132 |
+
act they want something done on the computer — launched, changed,
|
| 133 |
+
created, downloaded, closed, installed, shut down, automated.
|
| 134 |
+
Includes requests phrased politely or indirectly ("I could really
|
| 135 |
+
use my email open", "get rid of these windows").
|
| 136 |
+
converse they want information, opinion, or chat. Questions about facts,
|
| 137 |
+
status, or how something works.
|
| 138 |
+
wake they are only getting the assistant's attention.
|
| 139 |
+
mute they want silence, however phrased.
|
| 140 |
+
unmute they want the voice back.
|
| 141 |
+
cancel stop the current action.
|
| 142 |
+
|
| 143 |
+
When a sentence both asks and instructs, prefer act.
|
| 144 |
+
"""
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
def model_intent(text: str) -> Optional[Intent]:
|
| 148 |
+
try:
|
| 149 |
+
from core.brain_router import ask_router
|
| 150 |
+
rep = ask_router(
|
| 151 |
+
user_text=text,
|
| 152 |
+
system_prompt=_CLASSIFY,
|
| 153 |
+
context_block=f"SPEAKER SAID: {text}",
|
| 154 |
+
provider_order=["gemini", "local"],
|
| 155 |
+
)
|
| 156 |
+
raw = (getattr(rep, "text", "") or "").strip()
|
| 157 |
+
raw = re.sub(r"^```(?:json)?|```$", "", raw).strip()
|
| 158 |
+
m = re.search(r"\{.*\}", raw, re.S)
|
| 159 |
+
if not m:
|
| 160 |
+
return None
|
| 161 |
+
data = json.loads(m.group(0))
|
| 162 |
+
kind = str(data.get("intent", "")).lower().strip()
|
| 163 |
+
if kind not in INTENTS:
|
| 164 |
+
return None
|
| 165 |
+
conf = float(data.get("confidence", 0.6) or 0.6)
|
| 166 |
+
return Intent(kind, conf, strip_address(text), "model")
|
| 167 |
+
except Exception:
|
| 168 |
+
return None
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
# ── Heuristic fallback ──────────────────────────────────────────────────────
|
| 172 |
+
def heuristic_intent(text: str) -> Intent:
|
| 173 |
+
"""Used only when the model is unreachable.
|
| 174 |
+
|
| 175 |
+
Still deliberately broader than the old action_words list: it treats an
|
| 176 |
+
imperative opening as an instruction rather than requiring a listed verb.
|
| 177 |
+
"""
|
| 178 |
+
body = strip_address(text) or text
|
| 179 |
+
if _QUESTION.match(body):
|
| 180 |
+
return Intent("converse", 0.55, body, "default")
|
| 181 |
+
# An imperative sentence usually opens with a bare verb. Rather than list
|
| 182 |
+
# verbs, notice what an instruction is NOT: a question, or a statement
|
| 183 |
+
# opening with a pronoun/article.
|
| 184 |
+
if re.match(r"^\s*(?:i|you|he|she|they|we|it|the|a|an|this|that|there|my)\b",
|
| 185 |
+
body, re.I):
|
| 186 |
+
return Intent("converse", 0.5, body, "default")
|
| 187 |
+
return Intent("act", 0.5, body, "default")
|
| 188 |
+
|
| 189 |
+
|
| 190 |
+
def classify(text: str, allow_model: bool = True) -> Intent:
|
| 191 |
+
"""The one call every surface should use."""
|
| 192 |
+
fast = fast_intent(text)
|
| 193 |
+
if fast is not None:
|
| 194 |
+
return fast
|
| 195 |
+
if allow_model:
|
| 196 |
+
got = model_intent(text)
|
| 197 |
+
if got is not None:
|
| 198 |
+
return got
|
| 199 |
+
return heuristic_intent(text)
|
| 200 |
+
|
| 201 |
+
|
| 202 |
+
__all__ = ["classify", "Intent", "is_addressed", "strip_address",
|
| 203 |
+
"fast_intent", "model_intent", "heuristic_intent", "INTENTS"]
|
modules/omega_executor.py
ADDED
|
@@ -0,0 +1,667 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""modules/omega_executor.py — the unlimited task engine.
|
| 2 |
+
|
| 3 |
+
WHY THIS EXISTS
|
| 4 |
+
---------------
|
| 5 |
+
Every command path in this system used to end at a keyword table. core.commands
|
| 6 |
+
has an `instant_commands` dict and a long if/elif ladder; modules.task_authoring
|
| 7 |
+
handles exactly one application, because its app map is literally
|
| 8 |
+
|
| 9 |
+
{"notepad": ("notepad.exe", "notepad"), ...}
|
| 10 |
+
|
| 11 |
+
So "open notepad and write X then save" worked and *nothing else did*. Not
|
| 12 |
+
because the machine lacked the power — backend/tools/tool_registry.py already
|
| 13 |
+
exposes run_shell, open_app, write_file, browser control, media control and two
|
| 14 |
+
dozen more — but because the voice and phone paths could not reach any of it.
|
| 15 |
+
The powers were designed and never granted.
|
| 16 |
+
|
| 17 |
+
This module is the grant. It takes a request in plain speech, PLANS it against
|
| 18 |
+
the real capability catalog, and executes the plan step by step. Nothing here is
|
| 19 |
+
keyed to a verb list or an app name, so "restart the machine", "download this,
|
| 20 |
+
install it, run it ten minutes, close everything and shut down", and requests
|
| 21 |
+
nobody anticipated all travel the same road.
|
| 22 |
+
|
| 23 |
+
WHEN NOTHING FITS, IT WRITES THE CODE
|
| 24 |
+
-------------------------------------
|
| 25 |
+
A catalog is still a finite list, and the requirement is explicitly unlimited.
|
| 26 |
+
So the planner has a step kind the others don't: `python`. If no capability
|
| 27 |
+
matches, the model authors a short script for that step and the engine runs it,
|
| 28 |
+
feeding its output to the next step. That is the difference between "unlimited
|
| 29 |
+
within what we thought of" and unlimited.
|
| 30 |
+
|
| 31 |
+
IRREVERSIBLE ACTIONS
|
| 32 |
+
--------------------
|
| 33 |
+
Shutdown, restart, delete, format and their kin are executed — the user asked
|
| 34 |
+
for exactly that, in those words — but they are executed LAST in a plan and
|
| 35 |
+
announced first, with a cancellation window. A misheard word should not be able
|
| 36 |
+
to take the machine down with no way to catch it. Everything reversible runs
|
| 37 |
+
without ceremony.
|
| 38 |
+
"""
|
| 39 |
+
|
| 40 |
+
from __future__ import annotations
|
| 41 |
+
|
| 42 |
+
import asyncio
|
| 43 |
+
import json
|
| 44 |
+
import os
|
| 45 |
+
import re
|
| 46 |
+
import subprocess
|
| 47 |
+
import sys
|
| 48 |
+
import tempfile
|
| 49 |
+
import threading
|
| 50 |
+
import time
|
| 51 |
+
from dataclasses import dataclass, field
|
| 52 |
+
from typing import Any, Callable, Optional
|
| 53 |
+
|
| 54 |
+
# ── Tunables ────────────────────────────────────────────────────────────────
|
| 55 |
+
MAX_STEPS = 40 # a chain longer than this is almost certainly a
|
| 56 |
+
# planner loop, not a user intention
|
| 57 |
+
STEP_TIMEOUT = 900 # 15 min: "run it for ten minutes" must fit
|
| 58 |
+
PYTHON_TIMEOUT = 300
|
| 59 |
+
DESTRUCTIVE_GRACE = 8 # seconds to say "cancel that" before the machine goes
|
| 60 |
+
|
| 61 |
+
_LOG_LOCK = threading.RLock()
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def _log(msg: str) -> None:
|
| 65 |
+
with _LOG_LOCK:
|
| 66 |
+
try:
|
| 67 |
+
print(f"[omega_executor] {msg}", flush=True)
|
| 68 |
+
except Exception:
|
| 69 |
+
pass
|
| 70 |
+
|
| 71 |
+
|
| 72 |
+
# ── What counts as irreversible ─────────────────────────────────────────────
|
| 73 |
+
# Matched against the SHELL/PYTHON text actually about to run, not against the
|
| 74 |
+
# user's phrasing. Phrasing lies; the command does not.
|
| 75 |
+
_DESTRUCTIVE_PATTERNS = (
|
| 76 |
+
r"\bshutdown\b", r"\bstop-computer\b", r"\brestart-computer\b",
|
| 77 |
+
r"\blogoff\b", r"\bshutdown\.exe\b",
|
| 78 |
+
r"\bformat\b", r"\bdiskpart\b", r"\bmkfs\b",
|
| 79 |
+
r"\brd\s+/s\b", r"\brmdir\s+/s\b",
|
| 80 |
+
# Deletion, in any form, with or without flags.
|
| 81 |
+
#
|
| 82 |
+
# These used to require -Recurse or /q, which meant they only caught the
|
| 83 |
+
# deletion of whole DIRECTORY TREES. `Remove-Item -Force` on a single file
|
| 84 |
+
# sailed straight through unclassified — verified: a scratch file was
|
| 85 |
+
# permanently deleted with no deliberation and no cancel window, because
|
| 86 |
+
# "remove-item ... -recurse" simply did not match it.
|
| 87 |
+
#
|
| 88 |
+
# One irreplaceable file is not meaningfully less irreversible than a
|
| 89 |
+
# folder of them, so the flags are no longer part of the test.
|
| 90 |
+
r"\bremove-item\b", r"\bdel\b", r"\berase\b", r"\bunlink\b",
|
| 91 |
+
r"\bclear-content\b", r"\brm\s+-[a-z]*[rf]", r"\brm\s+",
|
| 92 |
+
r"\bcipher\s+/w\b", r"\bbcdedit\b", r"\breg\s+delete\b",
|
| 93 |
+
r"\bremove-partition\b", r"\bclear-disk\b",
|
| 94 |
+
)
|
| 95 |
+
|
| 96 |
+
|
| 97 |
+
def is_destructive(text: str) -> bool:
|
| 98 |
+
low = (text or "").lower()
|
| 99 |
+
return any(re.search(p, low) for p in _DESTRUCTIVE_PATTERNS)
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
# ── Result plumbing ─────────────────────────────────────────────────────────
|
| 103 |
+
@dataclass
|
| 104 |
+
class StepResult:
|
| 105 |
+
index: int
|
| 106 |
+
kind: str
|
| 107 |
+
summary: str
|
| 108 |
+
ok: bool
|
| 109 |
+
output: str = ""
|
| 110 |
+
error: str = ""
|
| 111 |
+
|
| 112 |
+
|
| 113 |
+
@dataclass
|
| 114 |
+
class RunResult:
|
| 115 |
+
ok: bool
|
| 116 |
+
spoken: str
|
| 117 |
+
steps: list[StepResult] = field(default_factory=list)
|
| 118 |
+
plan: list[dict] = field(default_factory=list)
|
| 119 |
+
cancelled: bool = False
|
| 120 |
+
|
| 121 |
+
def transcript(self) -> str:
|
| 122 |
+
lines = []
|
| 123 |
+
for s in self.steps:
|
| 124 |
+
mark = "ok " if s.ok else "FAIL"
|
| 125 |
+
lines.append(f" [{mark}] {s.index}. {s.summary}")
|
| 126 |
+
if s.error:
|
| 127 |
+
lines.append(f" {s.error[:300]}")
|
| 128 |
+
return "\n".join(lines)
|
| 129 |
+
|
| 130 |
+
|
| 131 |
+
# ── Capability catalog ──────────────────────────────────────────────────────
|
| 132 |
+
# Built by introspecting the REAL registry rather than by hand, so a tool added
|
| 133 |
+
# anywhere becomes usable by voice on the next run with no edit here. This is
|
| 134 |
+
# the mechanism that grants the already-designed-but-ungranted powers.
|
| 135 |
+
_CATALOG_CACHE: Optional[list[dict]] = None
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def capability_catalog(refresh: bool = False) -> list[dict]:
|
| 139 |
+
global _CATALOG_CACHE
|
| 140 |
+
if _CATALOG_CACHE is not None and not refresh:
|
| 141 |
+
return _CATALOG_CACHE
|
| 142 |
+
|
| 143 |
+
caps: list[dict] = []
|
| 144 |
+
try:
|
| 145 |
+
from backend.tools.tool_registry import TOOL_REGISTRY
|
| 146 |
+
import inspect
|
| 147 |
+
for name, fn in TOOL_REGISTRY.items():
|
| 148 |
+
try:
|
| 149 |
+
sig = inspect.signature(fn)
|
| 150 |
+
params = [
|
| 151 |
+
p.name for p in sig.parameters.values()
|
| 152 |
+
if p.name not in ("self", "args", "kwargs")
|
| 153 |
+
and p.kind not in (p.VAR_POSITIONAL, p.VAR_KEYWORD)
|
| 154 |
+
]
|
| 155 |
+
except Exception:
|
| 156 |
+
params = []
|
| 157 |
+
doc = (fn.__doc__ or "").strip().split("\n")[0][:160]
|
| 158 |
+
caps.append({"name": name, "params": params, "doc": doc})
|
| 159 |
+
except Exception as e:
|
| 160 |
+
_log(f"tool registry unavailable ({e}) — shell and python still work")
|
| 161 |
+
|
| 162 |
+
_CATALOG_CACHE = caps
|
| 163 |
+
return caps
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def _catalog_text() -> str:
|
| 167 |
+
caps = capability_catalog()
|
| 168 |
+
if not caps:
|
| 169 |
+
return "(no named tools reachable; use shell and python)"
|
| 170 |
+
return "\n".join(
|
| 171 |
+
f"- {c['name']}({', '.join(c['params'])})"
|
| 172 |
+
+ (f" — {c['doc']}" if c["doc"] else "")
|
| 173 |
+
for c in caps
|
| 174 |
+
)
|
| 175 |
+
|
| 176 |
+
|
| 177 |
+
# ── The planner ─────────────────────────────────────────────────────────────
|
| 178 |
+
_PLANNER_RULES = """You convert a spoken request into an executable plan for a
|
| 179 |
+
Windows machine. Return ONLY a JSON object, no prose, no code fences.
|
| 180 |
+
|
| 181 |
+
{"say": "<one short line to speak before starting>",
|
| 182 |
+
"steps": [ {"kind": "...", ...}, ... ]}
|
| 183 |
+
|
| 184 |
+
Step kinds:
|
| 185 |
+
|
| 186 |
+
{"kind":"tool","name":"<catalog name>","args":{...},"why":"<short summary>"}
|
| 187 |
+
Prefer this whenever a catalog tool fits.
|
| 188 |
+
|
| 189 |
+
{"kind":"shell","cmd":"<a real Windows command>","why":"..."}
|
| 190 |
+
PowerShell/cmd. Use for anything the catalog does not cover.
|
| 191 |
+
|
| 192 |
+
{"kind":"python","code":"<a complete python script>","why":"..."}
|
| 193 |
+
USE THIS WHEN NOTHING ELSE FITS. You are allowed and expected to write
|
| 194 |
+
new code for requests nobody anticipated. The script may import anything
|
| 195 |
+
installed, read $OMEGA_PREV for the previous step's output, and print its
|
| 196 |
+
result. Keep it self-contained.
|
| 197 |
+
|
| 198 |
+
{"kind":"wait","seconds":<number>,"why":"..."}
|
| 199 |
+
Real elapsed time — "run it for ten minutes" is {"kind":"wait","seconds":600}.
|
| 200 |
+
|
| 201 |
+
{"kind":"say","text":"..."}
|
| 202 |
+
Speak progress mid-chain.
|
| 203 |
+
|
| 204 |
+
Rules:
|
| 205 |
+
- Decompose chains fully and in order. "download X, install it, run it 10
|
| 206 |
+
minutes, close everything, shut down" is at least 5 steps, not 1.
|
| 207 |
+
- Irreversible steps (shutdown, restart, format, mass delete) go LAST.
|
| 208 |
+
- Never invent a tool name that is not in the catalog.
|
| 209 |
+
- Never ask the user a question; choose the sensible interpretation and act.
|
| 210 |
+
- Prefer one capable step over many timid ones, but never merge a wait.
|
| 211 |
+
"""
|
| 212 |
+
|
| 213 |
+
|
| 214 |
+
def _extract_json(raw: str) -> Optional[dict]:
|
| 215 |
+
"""Models wrap JSON in prose or fences no matter how firmly you ask."""
|
| 216 |
+
if not raw:
|
| 217 |
+
return None
|
| 218 |
+
txt = raw.strip()
|
| 219 |
+
txt = re.sub(r"^```(?:json)?", "", txt).strip()
|
| 220 |
+
txt = re.sub(r"```$", "", txt).strip()
|
| 221 |
+
try:
|
| 222 |
+
return json.loads(txt)
|
| 223 |
+
except Exception:
|
| 224 |
+
pass
|
| 225 |
+
# First balanced {...} block.
|
| 226 |
+
start = txt.find("{")
|
| 227 |
+
while start != -1:
|
| 228 |
+
depth, instr, esc = 0, False, False
|
| 229 |
+
for i in range(start, len(txt)):
|
| 230 |
+
ch = txt[i]
|
| 231 |
+
if esc:
|
| 232 |
+
esc = False
|
| 233 |
+
continue
|
| 234 |
+
if ch == "\\":
|
| 235 |
+
esc = True
|
| 236 |
+
continue
|
| 237 |
+
if ch == '"':
|
| 238 |
+
instr = not instr
|
| 239 |
+
continue
|
| 240 |
+
if instr:
|
| 241 |
+
continue
|
| 242 |
+
if ch == "{":
|
| 243 |
+
depth += 1
|
| 244 |
+
elif ch == "}":
|
| 245 |
+
depth -= 1
|
| 246 |
+
if depth == 0:
|
| 247 |
+
try:
|
| 248 |
+
return json.loads(txt[start:i + 1])
|
| 249 |
+
except Exception:
|
| 250 |
+
break
|
| 251 |
+
start = txt.find("{", start + 1)
|
| 252 |
+
return None
|
| 253 |
+
|
| 254 |
+
|
| 255 |
+
def plan_request(request: str, context: str = "") -> dict:
|
| 256 |
+
"""Turn plain speech into a step plan. Never raises."""
|
| 257 |
+
system = _PLANNER_RULES + "\n\nCATALOG:\n" + _catalog_text()
|
| 258 |
+
ctx = [f"Machine: Windows. Working dir: {os.getcwd()}"]
|
| 259 |
+
if context:
|
| 260 |
+
ctx.append(context)
|
| 261 |
+
ctx.append(f"REQUEST: {request}")
|
| 262 |
+
|
| 263 |
+
raw = ""
|
| 264 |
+
try:
|
| 265 |
+
from core.brain_router import ask_router
|
| 266 |
+
rep = ask_router(
|
| 267 |
+
user_text=request,
|
| 268 |
+
system_prompt=system,
|
| 269 |
+
context_block="\n".join(ctx),
|
| 270 |
+
provider_order=["gemini", "local"],
|
| 271 |
+
)
|
| 272 |
+
raw = getattr(rep, "text", "") or ""
|
| 273 |
+
except Exception as e:
|
| 274 |
+
_log(f"planner LLM unavailable: {e}")
|
| 275 |
+
|
| 276 |
+
plan = _extract_json(raw)
|
| 277 |
+
if not isinstance(plan, dict) or not isinstance(plan.get("steps"), list):
|
| 278 |
+
# No plan means no invention: fall back to running the literal request
|
| 279 |
+
# as a shell line, which is still better than the old silent refusal.
|
| 280 |
+
return {
|
| 281 |
+
"say": "Running that directly.",
|
| 282 |
+
"steps": [{"kind": "shell", "cmd": request, "why": request}],
|
| 283 |
+
"degraded": True,
|
| 284 |
+
}
|
| 285 |
+
|
| 286 |
+
steps = [s for s in plan["steps"] if isinstance(s, dict) and s.get("kind")]
|
| 287 |
+
plan["steps"] = steps[:MAX_STEPS]
|
| 288 |
+
return plan
|
| 289 |
+
|
| 290 |
+
|
| 291 |
+
# ── Step execution ──────────────────────────────────────────────────────────
|
| 292 |
+
def _run_shell(cmd: str, timeout: int = STEP_TIMEOUT) -> tuple[bool, str, str]:
|
| 293 |
+
try:
|
| 294 |
+
creation = 0
|
| 295 |
+
if os.name == "nt":
|
| 296 |
+
creation = getattr(subprocess, "CREATE_NO_WINDOW", 0)
|
| 297 |
+
proc = subprocess.run(
|
| 298 |
+
["powershell", "-NoProfile", "-NonInteractive", "-Command", cmd],
|
| 299 |
+
capture_output=True, text=True, timeout=timeout,
|
| 300 |
+
creationflags=creation,
|
| 301 |
+
)
|
| 302 |
+
out = (proc.stdout or "").strip()
|
| 303 |
+
err = (proc.stderr or "").strip()
|
| 304 |
+
return proc.returncode == 0, out, err
|
| 305 |
+
except subprocess.TimeoutExpired:
|
| 306 |
+
return False, "", f"timed out after {timeout}s"
|
| 307 |
+
except Exception as e:
|
| 308 |
+
return False, "", str(e)
|
| 309 |
+
|
| 310 |
+
|
| 311 |
+
def _run_python(code: str, prev: str = "") -> tuple[bool, str, str]:
|
| 312 |
+
"""Run model-authored code in a child process.
|
| 313 |
+
|
| 314 |
+
A child, not exec() in-process: authored code that throws, blocks or calls
|
| 315 |
+
sys.exit must not be able to take JARVIS down with it.
|
| 316 |
+
"""
|
| 317 |
+
path = ""
|
| 318 |
+
try:
|
| 319 |
+
fd, path = tempfile.mkstemp(suffix=".py", prefix="omega_step_")
|
| 320 |
+
with os.fdopen(fd, "w", encoding="utf-8") as fh:
|
| 321 |
+
fh.write(code)
|
| 322 |
+
env = dict(os.environ)
|
| 323 |
+
env["OMEGA_PREV"] = (prev or "")[:8000]
|
| 324 |
+
env["PYTHONIOENCODING"] = "utf-8"
|
| 325 |
+
creation = getattr(subprocess, "CREATE_NO_WINDOW", 0) if os.name == "nt" else 0
|
| 326 |
+
proc = subprocess.run(
|
| 327 |
+
[sys.executable, path], capture_output=True, text=True,
|
| 328 |
+
timeout=PYTHON_TIMEOUT, env=env, creationflags=creation,
|
| 329 |
+
)
|
| 330 |
+
return (proc.returncode == 0,
|
| 331 |
+
(proc.stdout or "").strip(),
|
| 332 |
+
(proc.stderr or "").strip())
|
| 333 |
+
except subprocess.TimeoutExpired:
|
| 334 |
+
return False, "", f"script timed out after {PYTHON_TIMEOUT}s"
|
| 335 |
+
except Exception as e:
|
| 336 |
+
return False, "", str(e)
|
| 337 |
+
finally:
|
| 338 |
+
if path:
|
| 339 |
+
try:
|
| 340 |
+
os.unlink(path)
|
| 341 |
+
except Exception:
|
| 342 |
+
pass
|
| 343 |
+
|
| 344 |
+
|
| 345 |
+
def _run_tool(name: str, args: dict) -> tuple[bool, str, str]:
|
| 346 |
+
try:
|
| 347 |
+
from backend.tools.tool_registry import TOOL_REGISTRY
|
| 348 |
+
except Exception as e:
|
| 349 |
+
return False, "", f"tool registry unavailable: {e}"
|
| 350 |
+
fn = TOOL_REGISTRY.get(name)
|
| 351 |
+
if fn is None:
|
| 352 |
+
return False, "", f"no such tool: {name}"
|
| 353 |
+
try:
|
| 354 |
+
res = fn(**(args or {}))
|
| 355 |
+
if asyncio.iscoroutine(res):
|
| 356 |
+
try:
|
| 357 |
+
asyncio.get_running_loop()
|
| 358 |
+
except RuntimeError:
|
| 359 |
+
res = asyncio.run(res)
|
| 360 |
+
else:
|
| 361 |
+
# Already inside a loop (backend request): use a private one in
|
| 362 |
+
# a worker thread rather than fighting the live loop.
|
| 363 |
+
box: dict[str, Any] = {}
|
| 364 |
+
|
| 365 |
+
def _worker():
|
| 366 |
+
try:
|
| 367 |
+
box["v"] = asyncio.run(res)
|
| 368 |
+
except Exception as ex:
|
| 369 |
+
box["e"] = ex
|
| 370 |
+
|
| 371 |
+
t = threading.Thread(target=_worker, daemon=True)
|
| 372 |
+
t.start()
|
| 373 |
+
t.join(STEP_TIMEOUT)
|
| 374 |
+
if "e" in box:
|
| 375 |
+
return False, "", str(box["e"])
|
| 376 |
+
res = box.get("v")
|
| 377 |
+
return True, ("" if res is None else str(res))[:4000], ""
|
| 378 |
+
except Exception as e:
|
| 379 |
+
return False, "", str(e)
|
| 380 |
+
|
| 381 |
+
|
| 382 |
+
# ── The engine ──────────────────────────────────────────────────────────────
|
| 383 |
+
def execute(request: str,
|
| 384 |
+
speak: Optional[Callable[[str], None]] = None,
|
| 385 |
+
context: str = "",
|
| 386 |
+
allow_destructive: bool = True,
|
| 387 |
+
confirm: Optional[Callable[[str], bool]] = None,
|
| 388 |
+
already_deliberated: bool = False) -> RunResult:
|
| 389 |
+
"""Do whatever was asked. The single entry point for every surface.
|
| 390 |
+
|
| 391 |
+
`already_deliberated` is set by modules.deferred_intent when it re-runs a
|
| 392 |
+
held order. Without it the order would be weighed a second time here and
|
| 393 |
+
could be deferred again by the same reasoning that just released it — an
|
| 394 |
+
order could bounce between the two layers forever and never actually run.
|
| 395 |
+
The cancel window is still honoured; only the judgement is skipped.
|
| 396 |
+
"""
|
| 397 |
+
say = speak or (lambda t: _log(f"say: {t}"))
|
| 398 |
+
|
| 399 |
+
plan = plan_request(request, context)
|
| 400 |
+
steps = plan.get("steps") or []
|
| 401 |
+
opening = (plan.get("say") or "").strip()
|
| 402 |
+
if opening:
|
| 403 |
+
say(opening)
|
| 404 |
+
|
| 405 |
+
results: list[StepResult] = []
|
| 406 |
+
prev_output = ""
|
| 407 |
+
ok_all = True
|
| 408 |
+
|
| 409 |
+
for i, step in enumerate(steps, 1):
|
| 410 |
+
kind = str(step.get("kind", "")).lower()
|
| 411 |
+
why = str(step.get("why") or step.get("text") or kind)[:180]
|
| 412 |
+
|
| 413 |
+
# Everything this step will actually run, whichever shape it arrived in.
|
| 414 |
+
#
|
| 415 |
+
# This used to be `step["cmd"] or step["code"]`, which silently exempted
|
| 416 |
+
# TOOL steps: the planner is free to express a deletion as
|
| 417 |
+
# {"kind":"tool","name":"run_shell","args":{"cmd":"Remove-Item ..."}},
|
| 418 |
+
# and then the command lives in args, step["cmd"] is empty, and
|
| 419 |
+
# is_destructive("") is False. Verified the hard way — a scratch file was
|
| 420 |
+
# permanently deleted with no deliberation and no cancel window because
|
| 421 |
+
# the plan happened to use the tool form rather than a shell step.
|
| 422 |
+
#
|
| 423 |
+
# The tool NAME matters too: run_shell carrying a shutdown is a shutdown
|
| 424 |
+
# regardless of how the JSON is shaped.
|
| 425 |
+
payload = " ".join(str(x) for x in (
|
| 426 |
+
step.get("cmd") or "",
|
| 427 |
+
step.get("code") or "",
|
| 428 |
+
step.get("name") or "",
|
| 429 |
+
json.dumps(step.get("args") or {}),
|
| 430 |
+
))
|
| 431 |
+
if kind in ("shell", "python", "tool") and is_destructive(payload):
|
| 432 |
+
if not allow_destructive:
|
| 433 |
+
results.append(StepResult(i, kind, why, False,
|
| 434 |
+
error="irreversible step refused"))
|
| 435 |
+
ok_all = False
|
| 436 |
+
continue
|
| 437 |
+
|
| 438 |
+
# Think about it properly first. This is NOT a permission check —
|
| 439 |
+
# the order is already given and will be carried out. It is the
|
| 440 |
+
# judgement a person applies before pulling a plug: is something
|
| 441 |
+
# mid-flight that this would wreck? If so, hold and say why, and
|
| 442 |
+
# modules.deferred_intent will raise it again in a few minutes and
|
| 443 |
+
# eventually go ahead regardless. Holding is a timer, not a refusal.
|
| 444 |
+
if not already_deliberated:
|
| 445 |
+
try:
|
| 446 |
+
from modules.deferred_intent import deliberate, defer
|
| 447 |
+
verdict = deliberate(request)
|
| 448 |
+
if not verdict.proceed:
|
| 449 |
+
held = defer(request, verdict.reason)
|
| 450 |
+
say(verdict.speak
|
| 451 |
+
or f"Holding off on that — {verdict.reason}. "
|
| 452 |
+
f"I'll come back to it.")
|
| 453 |
+
results.append(StepResult(
|
| 454 |
+
i, kind, why, True,
|
| 455 |
+
output=f"deferred (#{held}): {verdict.reason}"))
|
| 456 |
+
return RunResult(True, "", results, steps)
|
| 457 |
+
except Exception as e:
|
| 458 |
+
_log(f"deliberation unavailable ({e}); order stands")
|
| 459 |
+
|
| 460 |
+
# The cancel window. Deliberately kept: a misheard word should have
|
| 461 |
+
# one last chance to be caught before the machine goes down.
|
| 462 |
+
say(f"{why}. Say cancel now if that's wrong.")
|
| 463 |
+
if confirm is not None:
|
| 464 |
+
if not confirm(why):
|
| 465 |
+
results.append(StepResult(i, kind, why, False,
|
| 466 |
+
error="cancelled by user"))
|
| 467 |
+
return RunResult(False, "Cancelled.", results, steps, True)
|
| 468 |
+
else:
|
| 469 |
+
time.sleep(DESTRUCTIVE_GRACE)
|
| 470 |
+
|
| 471 |
+
if kind == "say":
|
| 472 |
+
text = str(step.get("text") or why)
|
| 473 |
+
say(text)
|
| 474 |
+
results.append(StepResult(i, kind, text[:120], True))
|
| 475 |
+
continue
|
| 476 |
+
|
| 477 |
+
if kind == "wait":
|
| 478 |
+
try:
|
| 479 |
+
secs = float(step.get("seconds") or 0)
|
| 480 |
+
except Exception:
|
| 481 |
+
secs = 0.0
|
| 482 |
+
secs = max(0.0, min(secs, STEP_TIMEOUT))
|
| 483 |
+
results.append(StepResult(i, kind, f"waited {int(secs)}s", True))
|
| 484 |
+
time.sleep(secs)
|
| 485 |
+
continue
|
| 486 |
+
|
| 487 |
+
if kind == "tool":
|
| 488 |
+
good, out, err = _run_tool(str(step.get("name") or ""),
|
| 489 |
+
step.get("args") or {})
|
| 490 |
+
elif kind == "python":
|
| 491 |
+
good, out, err = _run_python(str(step.get("code") or ""), prev_output)
|
| 492 |
+
else: # shell, and anything unrecognised is treated as one
|
| 493 |
+
good, out, err = _run_shell(str(step.get("cmd") or ""))
|
| 494 |
+
|
| 495 |
+
# ── IMMEDIATE SELF-REPAIR ────────────────────────────────────────────
|
| 496 |
+
#
|
| 497 |
+
# A failing step used to end the chain. But most failures are not "this
|
| 498 |
+
# is impossible" — they are a wrong path, a missing quote, a cmdlet that
|
| 499 |
+
# does not exist on this box, a tool called with the wrong argument.
|
| 500 |
+
# A person would not abandon the task; they would look at the error, fix
|
| 501 |
+
# the command, and run it again, all within a few seconds.
|
| 502 |
+
#
|
| 503 |
+
# So that is what happens. The error text and the failed command go back
|
| 504 |
+
# to the model, which returns a corrected version, and it is retried
|
| 505 |
+
# immediately. The operator asked for the thing to happen NOW, and this
|
| 506 |
+
# is what makes it happen now rather than after a bug report.
|
| 507 |
+
#
|
| 508 |
+
# The lasting fix is a separate, slower matter — recorded and handled
|
| 509 |
+
# later, because stopping to redesign something mid-task is exactly the
|
| 510 |
+
# wrong moment for it.
|
| 511 |
+
if not good and kind in ("shell", "python", "tool"):
|
| 512 |
+
for attempt in range(1, MAX_REPAIRS + 1):
|
| 513 |
+
fixed = _repair_step(step, err, request)
|
| 514 |
+
if not fixed:
|
| 515 |
+
break
|
| 516 |
+
_log(f"step {i} repair attempt {attempt}")
|
| 517 |
+
if kind == "tool":
|
| 518 |
+
good, out, err = _run_tool(str(fixed.get("name") or ""),
|
| 519 |
+
fixed.get("args") or {})
|
| 520 |
+
elif kind == "python":
|
| 521 |
+
good, out, err = _run_python(str(fixed.get("code") or ""),
|
| 522 |
+
prev_output)
|
| 523 |
+
else:
|
| 524 |
+
good, out, err = _run_shell(str(fixed.get("cmd") or ""))
|
| 525 |
+
if good:
|
| 526 |
+
why = f"{why} (repaired)"
|
| 527 |
+
_remember_repair(request, step, fixed, err_before=err)
|
| 528 |
+
break
|
| 529 |
+
step = fixed # each attempt learns from the last error
|
| 530 |
+
|
| 531 |
+
results.append(StepResult(i, kind, why, good, out, err))
|
| 532 |
+
if out:
|
| 533 |
+
prev_output = out
|
| 534 |
+
if not good:
|
| 535 |
+
ok_all = False
|
| 536 |
+
_log(f"step {i} failed after repair attempts: {err[:200]}")
|
| 537 |
+
# A broken link means the rest of the chain is operating on false
|
| 538 |
+
# premises, so stop rather than plough on.
|
| 539 |
+
break
|
| 540 |
+
|
| 541 |
+
if ok_all:
|
| 542 |
+
spoken = _summarise_success(request, results)
|
| 543 |
+
else:
|
| 544 |
+
bad = next((r for r in results if not r.ok), None)
|
| 545 |
+
spoken = (f"I got through {len(results) - 1} of {len(steps)} steps, then "
|
| 546 |
+
f"{bad.summary.lower() if bad else 'a step'} failed."
|
| 547 |
+
if bad else "That didn't complete.")
|
| 548 |
+
return RunResult(ok_all, spoken, results, steps)
|
| 549 |
+
|
| 550 |
+
|
| 551 |
+
# ── Self-repair ─────────────────────────────────────────────────────────────
|
| 552 |
+
MAX_REPAIRS = 2 # two corrected attempts, then admit it is stuck
|
| 553 |
+
_REPAIR_LOG = os.path.join(
|
| 554 |
+
os.environ.get("JARVIS_APP_DATA_DIR") or os.path.join(
|
| 555 |
+
os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".friday_data"),
|
| 556 |
+
"repairs.json")
|
| 557 |
+
|
| 558 |
+
_REPAIR_RULES = """A command you just ran FAILED. Fix it and return the
|
| 559 |
+
corrected step, as JSON only, in exactly the shape you were given.
|
| 560 |
+
|
| 561 |
+
You are not redesigning anything. You are getting this ONE command to work on
|
| 562 |
+
this machine, right now, from the error it produced. Read the error literally:
|
| 563 |
+
a path that does not exist, a cmdlet that is not available, a quoting mistake,
|
| 564 |
+
a wrong argument name, a missing flag.
|
| 565 |
+
|
| 566 |
+
Return the same "kind" with corrected fields:
|
| 567 |
+
shell -> {"kind":"shell","cmd":"<fixed command>"}
|
| 568 |
+
python -> {"kind":"python","code":"<fixed script>"}
|
| 569 |
+
tool -> {"kind":"tool","name":"<tool>","args":{...}}
|
| 570 |
+
|
| 571 |
+
If the approach itself cannot work, switch to one that can — a shell step
|
| 572 |
+
instead of a tool, or a python step that does the job directly. Returning
|
| 573 |
+
something that will fail the same way is worse than useless.
|
| 574 |
+
"""
|
| 575 |
+
|
| 576 |
+
|
| 577 |
+
def _repair_step(step: dict, error: str, original_request: str) -> Optional[dict]:
|
| 578 |
+
"""Ask for a corrected version of a step that just failed."""
|
| 579 |
+
if not error:
|
| 580 |
+
return None
|
| 581 |
+
try:
|
| 582 |
+
from core.brain_router import ask_router
|
| 583 |
+
ctx = (f"ORIGINAL REQUEST: {original_request}\n"
|
| 584 |
+
f"STEP THAT FAILED: {json.dumps(step)[:1500]}\n"
|
| 585 |
+
f"ERROR: {error[:1200]}\n"
|
| 586 |
+
f"Machine: Windows. Working dir: {os.getcwd()}")
|
| 587 |
+
rep = ask_router(user_text="Fix this command.",
|
| 588 |
+
system_prompt=_REPAIR_RULES,
|
| 589 |
+
context_block=ctx,
|
| 590 |
+
provider_order=["gemini", "local"])
|
| 591 |
+
fixed = _extract_json(getattr(rep, "text", "") or "")
|
| 592 |
+
if not isinstance(fixed, dict) or not fixed.get("kind"):
|
| 593 |
+
return None
|
| 594 |
+
# A "fix" identical to what just failed is not a fix.
|
| 595 |
+
if (str(fixed.get("cmd") or "") == str(step.get("cmd") or "")
|
| 596 |
+
and str(fixed.get("code") or "") == str(step.get("code") or "")
|
| 597 |
+
and str(fixed.get("name") or "") == str(step.get("name") or "")):
|
| 598 |
+
return None
|
| 599 |
+
# Never let a repair quietly escalate into something irreversible.
|
| 600 |
+
payload = " ".join(str(x) for x in (fixed.get("cmd") or "",
|
| 601 |
+
fixed.get("code") or "",
|
| 602 |
+
fixed.get("name") or "",
|
| 603 |
+
json.dumps(fixed.get("args") or {})))
|
| 604 |
+
if is_destructive(payload) and not is_destructive(
|
| 605 |
+
" ".join(str(y) for y in (step.get("cmd") or "",
|
| 606 |
+
step.get("code") or ""))):
|
| 607 |
+
_log("rejected a repair that introduced an irreversible action")
|
| 608 |
+
return None
|
| 609 |
+
return fixed
|
| 610 |
+
except Exception as e:
|
| 611 |
+
_log(f"repair unavailable: {e}")
|
| 612 |
+
return None
|
| 613 |
+
|
| 614 |
+
|
| 615 |
+
def _remember_repair(request: str, broken: dict, fixed: dict,
|
| 616 |
+
err_before: str = "") -> None:
|
| 617 |
+
"""Record what needed fixing, for a proper fix later.
|
| 618 |
+
|
| 619 |
+
The immediate repair gets the operator unblocked, which is the priority.
|
| 620 |
+
But a command that had to be corrected at runtime will need correcting
|
| 621 |
+
again next time unless something changes — so the evidence is kept, rather
|
| 622 |
+
than the problem being silently papered over on every single run.
|
| 623 |
+
"""
|
| 624 |
+
try:
|
| 625 |
+
os.makedirs(os.path.dirname(_REPAIR_LOG), exist_ok=True)
|
| 626 |
+
try:
|
| 627 |
+
with open(_REPAIR_LOG, "r", encoding="utf-8") as fh:
|
| 628 |
+
data = json.load(fh)
|
| 629 |
+
if not isinstance(data, list):
|
| 630 |
+
data = []
|
| 631 |
+
except Exception:
|
| 632 |
+
data = []
|
| 633 |
+
data.append({
|
| 634 |
+
"at": time.time(),
|
| 635 |
+
"request": request[:300],
|
| 636 |
+
"broken": {k: str(v)[:400] for k, v in broken.items()},
|
| 637 |
+
"fixed": {k: str(v)[:400] for k, v in fixed.items()},
|
| 638 |
+
"error": err_before[:500],
|
| 639 |
+
})
|
| 640 |
+
with open(_REPAIR_LOG, "w", encoding="utf-8") as fh:
|
| 641 |
+
json.dump(data[-100:], fh, indent=2)
|
| 642 |
+
except Exception:
|
| 643 |
+
pass
|
| 644 |
+
|
| 645 |
+
|
| 646 |
+
def pending_repairs() -> list[dict]:
|
| 647 |
+
"""Runtime corrections so far — the queue for real, lasting fixes."""
|
| 648 |
+
try:
|
| 649 |
+
with open(_REPAIR_LOG, "r", encoding="utf-8") as fh:
|
| 650 |
+
data = json.load(fh)
|
| 651 |
+
return data if isinstance(data, list) else []
|
| 652 |
+
except Exception:
|
| 653 |
+
return []
|
| 654 |
+
|
| 655 |
+
|
| 656 |
+
def _summarise_success(request: str, results: list[StepResult]) -> str:
|
| 657 |
+
done = [r for r in results if r.ok and r.kind not in ("say",)]
|
| 658 |
+
if not done:
|
| 659 |
+
return "Nothing to do there."
|
| 660 |
+
if len(done) == 1:
|
| 661 |
+
return f"Done — {done[0].summary}."
|
| 662 |
+
return f"Done. {len(done)} steps: " + "; ".join(r.summary for r in done[:4]) + (
|
| 663 |
+
"…" if len(done) > 4 else ".")
|
| 664 |
+
|
| 665 |
+
|
| 666 |
+
__all__ = ["execute", "plan_request", "capability_catalog", "is_destructive",
|
| 667 |
+
"RunResult", "StepResult"]
|
modules/process_utils.py
CHANGED
|
@@ -58,3 +58,47 @@ def run_no_window(
|
|
| 58 |
kwargs["creationflags"] = windows_creationflags_no_window()
|
| 59 |
return subprocess.run(list(args), **kwargs)
|
| 60 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 58 |
kwargs["creationflags"] = windows_creationflags_no_window()
|
| 59 |
return subprocess.run(list(args), **kwargs)
|
| 60 |
|
| 61 |
+
|
| 62 |
+
_console_suppressed = False
|
| 63 |
+
|
| 64 |
+
|
| 65 |
+
def suppress_console_windows() -> None:
|
| 66 |
+
"""Stop every child process in THIS process from flashing a console.
|
| 67 |
+
|
| 68 |
+
The helpers above only work where someone remembered to call them, and
|
| 69 |
+
across modules/ there are around forty bare `subprocess.run`/`Popen` calls
|
| 70 |
+
that do not. Most are harmless one-offs, but the watch loops shell out
|
| 71 |
+
constantly — `netstat -ano` for intrusion and connection checks, PowerShell
|
| 72 |
+
for disk health and driver anomalies, `wmic` for process listings — every
|
| 73 |
+
few seconds, forever.
|
| 74 |
+
|
| 75 |
+
Each of those is a CONSOLE program. Launched from a windowless parent
|
| 76 |
+
(pythonw, which is how the daemon, the relay and the tray all run), Windows
|
| 77 |
+
has no console to attach them to, so it creates one — and the window
|
| 78 |
+
appears and vanishes. That is the flashing: not one bug, one per call site.
|
| 79 |
+
|
| 80 |
+
Fixing forty call sites individually invites the forty-first. This sets the
|
| 81 |
+
default once, for the whole process: any child created WITHOUT explicit
|
| 82 |
+
creationflags gets CREATE_NO_WINDOW. A caller that passes its own flags is
|
| 83 |
+
left completely alone, so nothing that deliberately wants a window loses it.
|
| 84 |
+
|
| 85 |
+
Call this early in any entry point that runs without a console.
|
| 86 |
+
"""
|
| 87 |
+
global _console_suppressed
|
| 88 |
+
if _console_suppressed or os.name != "nt":
|
| 89 |
+
return
|
| 90 |
+
|
| 91 |
+
CREATE_NO_WINDOW = 0x08000000
|
| 92 |
+
_original_init = subprocess.Popen.__init__
|
| 93 |
+
|
| 94 |
+
def _init(self, *args, **kwargs):
|
| 95 |
+
# Only fill in a default. Never override an explicit choice, and never
|
| 96 |
+
# add DETACHED_PROCESS here — that would break callers who read the
|
| 97 |
+
# child's stdout, which several of these modules do.
|
| 98 |
+
if not kwargs.get("creationflags"):
|
| 99 |
+
kwargs["creationflags"] = CREATE_NO_WINDOW
|
| 100 |
+
return _original_init(self, *args, **kwargs)
|
| 101 |
+
|
| 102 |
+
subprocess.Popen.__init__ = _init
|
| 103 |
+
_console_suppressed = True
|
| 104 |
+
|
modules/security_review.py
CHANGED
|
@@ -114,8 +114,19 @@ def run_security_review(target: str = None) -> str:
|
|
| 114 |
source_context = _gather_target_context(target_path)
|
| 115 |
|
| 116 |
# Build the final prompt
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 117 |
prompt = (
|
| 118 |
-
f"You are
|
| 119 |
f"Target System Scope:\n{target_path}\n\n"
|
| 120 |
f"Discovered Source Code & Configurations:\n{source_context}\n\n"
|
| 121 |
f"Please begin your analysis and output the final markdown report."
|
|
|
|
| 114 |
source_context = _gather_target_context(target_path)
|
| 115 |
|
| 116 |
# Build the final prompt
|
| 117 |
+
#
|
| 118 |
+
# The persona is read, not assumed: this said "You are JARVIS" outright,
|
| 119 |
+
# so a security review run while FRIDAY was on duty was conducted in the
|
| 120 |
+
# other assistant's name. The audit is the same either way, but the
|
| 121 |
+
# report it writes is spoken back to the user in first person.
|
| 122 |
+
try:
|
| 123 |
+
from modules.assistant_identity import get_assistant_name
|
| 124 |
+
_who = get_assistant_name()
|
| 125 |
+
except Exception:
|
| 126 |
+
_who = "JARVIS"
|
| 127 |
+
|
| 128 |
prompt = (
|
| 129 |
+
f"You are {_who}, conducting an exhaustive security audit based on the provided framework.\n\n"
|
| 130 |
f"Target System Scope:\n{target_path}\n\n"
|
| 131 |
f"Discovered Source Code & Configurations:\n{source_context}\n\n"
|
| 132 |
f"Please begin your analysis and output the final markdown report."
|
modules/task_authoring.py
ADDED
|
@@ -0,0 +1,360 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
"""Multi-step desktop tasks: open an app, type into it, save the file.
|
| 4 |
+
|
| 5 |
+
WHY THIS EXISTS
|
| 6 |
+
---------------
|
| 7 |
+
"open notepad and write JARVIS then save to desktop" used to produce:
|
| 8 |
+
|
| 9 |
+
Executing 2 linked tasks.
|
| 10 |
+
Launching notepad and write jarvis. <- one app name, not two steps
|
| 11 |
+
Got it: save to desktop. Tell me the outcome you want.
|
| 12 |
+
|
| 13 |
+
The chain splitter in core.commands splits on "then" only, so "open notepad and
|
| 14 |
+
write JARVIS" stayed a single instruction and was handed to the app launcher as
|
| 15 |
+
if "notepad and write jarvis" were a program. Nothing could type, and nothing
|
| 16 |
+
could save — there was no handler for either verb.
|
| 17 |
+
|
| 18 |
+
This module does the three things that sentence actually asks for, in order,
|
| 19 |
+
against the real window: launch, type, save-as. It is deliberately narrow. It
|
| 20 |
+
does NOT try to be a general agent; it recognises a small, explicit vocabulary
|
| 21 |
+
and refuses anything outside it rather than guessing and reporting success.
|
| 22 |
+
|
| 23 |
+
WHY UI AUTOMATION AND NOT A FILE WRITE
|
| 24 |
+
--------------------------------------
|
| 25 |
+
"open notepad and write X then save" could be faked by writing a .txt directly
|
| 26 |
+
and never opening anything. That would pass a naive test and be a lie: the user
|
| 27 |
+
asked to SEE it happen, and the same mechanism has to work for editors that have
|
| 28 |
+
no file format we can synthesise. So this drives the real window with real
|
| 29 |
+
keystrokes.
|
| 30 |
+
"""
|
| 31 |
+
|
| 32 |
+
import os
|
| 33 |
+
import re
|
| 34 |
+
import subprocess
|
| 35 |
+
import time
|
| 36 |
+
from dataclasses import dataclass, field
|
| 37 |
+
from pathlib import Path
|
| 38 |
+
|
| 39 |
+
try:
|
| 40 |
+
import keyboard # type: ignore
|
| 41 |
+
except Exception: # pragma: no cover - depends on host packages
|
| 42 |
+
keyboard = None
|
| 43 |
+
|
| 44 |
+
try:
|
| 45 |
+
import win32gui # type: ignore
|
| 46 |
+
except Exception: # pragma: no cover
|
| 47 |
+
win32gui = None
|
| 48 |
+
|
| 49 |
+
|
| 50 |
+
# Programs we know how to launch by a spoken name, and the window title
|
| 51 |
+
# fragment that proves the right one came up.
|
| 52 |
+
KNOWN_APPS: dict[str, tuple[str, str]] = {
|
| 53 |
+
"notepad": ("notepad.exe", "notepad"),
|
| 54 |
+
"wordpad": ("write.exe", "wordpad"),
|
| 55 |
+
"word": ("winword.exe", "word"),
|
| 56 |
+
"excel": ("excel.exe", "excel"),
|
| 57 |
+
"paint": ("mspaint.exe", "paint"),
|
| 58 |
+
"calculator": ("calc.exe", "calculator"),
|
| 59 |
+
"explorer": ("explorer.exe", "explorer"),
|
| 60 |
+
"chrome": ("chrome.exe", "chrome"),
|
| 61 |
+
"edge": ("msedge.exe", "edge"),
|
| 62 |
+
}
|
| 63 |
+
|
| 64 |
+
# Where "save to X" can mean.
|
| 65 |
+
KNOWN_FOLDERS: dict[str, Path] = {
|
| 66 |
+
"desktop": Path.home() / "Desktop",
|
| 67 |
+
"documents": Path.home() / "Documents",
|
| 68 |
+
"downloads": Path.home() / "Downloads",
|
| 69 |
+
"pictures": Path.home() / "Pictures",
|
| 70 |
+
}
|
| 71 |
+
|
| 72 |
+
|
| 73 |
+
@dataclass
|
| 74 |
+
class Step:
|
| 75 |
+
kind: str # "open" | "type" | "save"
|
| 76 |
+
value: str = ""
|
| 77 |
+
folder: Path | None = None
|
| 78 |
+
filename: str | None = None
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
@dataclass
|
| 82 |
+
class TaskResult:
|
| 83 |
+
ok: bool
|
| 84 |
+
steps_done: int
|
| 85 |
+
messages: list[str] = field(default_factory=list)
|
| 86 |
+
|
| 87 |
+
@property
|
| 88 |
+
def message(self) -> str:
|
| 89 |
+
return " ".join(self.messages) if self.messages else ""
|
| 90 |
+
|
| 91 |
+
|
| 92 |
+
# --------------------------------------------------------------------------- #
|
| 93 |
+
# parsing
|
| 94 |
+
# --------------------------------------------------------------------------- #
|
| 95 |
+
def _split_clauses(text: str) -> list[str]:
|
| 96 |
+
"""Break on the connectives people actually use, not just "then".
|
| 97 |
+
|
| 98 |
+
core.commands.split_chain only knows "then", which is why "open notepad and
|
| 99 |
+
write JARVIS" survived as one clause. Splitting on "and"/","/"then" as
|
| 100 |
+
SEPARATORS BETWEEN VERBS keeps "write hello and goodbye" intact, because the
|
| 101 |
+
split only happens where the next clause starts with a known verb.
|
| 102 |
+
"""
|
| 103 |
+
parts = re.split(r"\s*(?:,|\bthen\b|\band\b|\bafter that\b)\s*", text, flags=re.I)
|
| 104 |
+
verbs = ("open", "launch", "start", "run", "write", "type", "enter", "save", "store")
|
| 105 |
+
clauses: list[str] = []
|
| 106 |
+
for raw in parts:
|
| 107 |
+
piece = raw.strip()
|
| 108 |
+
if not piece:
|
| 109 |
+
continue
|
| 110 |
+
first = piece.split()[0].lower()
|
| 111 |
+
if first in verbs or not clauses:
|
| 112 |
+
clauses.append(piece)
|
| 113 |
+
else:
|
| 114 |
+
# A fragment that does not begin a new action belongs to the one
|
| 115 |
+
# before it: "write hello" + "goodbye" -> "write hello and goodbye".
|
| 116 |
+
clauses[-1] = f"{clauses[-1]} and {piece}"
|
| 117 |
+
return clauses
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def parse_task(text: str) -> list[Step]:
|
| 121 |
+
"""Turn a sentence into ordered steps. Returns [] when it is not our shape."""
|
| 122 |
+
steps: list[Step] = []
|
| 123 |
+
for clause in _split_clauses(text or ""):
|
| 124 |
+
low = clause.lower().strip()
|
| 125 |
+
|
| 126 |
+
m = re.match(r"^(?:open|launch|start|run)\s+(.+)$", low)
|
| 127 |
+
if m:
|
| 128 |
+
target = m.group(1).strip()
|
| 129 |
+
# Longest known app name that appears, so "open notepad app" works.
|
| 130 |
+
app = next((name for name in sorted(KNOWN_APPS, key=len, reverse=True)
|
| 131 |
+
if name in target), None)
|
| 132 |
+
steps.append(Step("open", app or target))
|
| 133 |
+
continue
|
| 134 |
+
|
| 135 |
+
m = re.match(r"^(?:write|type|enter)\s+(.+)$", clause, flags=re.I)
|
| 136 |
+
if m:
|
| 137 |
+
payload = m.group(1).strip()
|
| 138 |
+
# Keep the user's capitalisation: they asked for "JARVIS", not "jarvis".
|
| 139 |
+
payload = payload.strip("\"'")
|
| 140 |
+
steps.append(Step("type", payload))
|
| 141 |
+
continue
|
| 142 |
+
|
| 143 |
+
m = re.match(r"^(?:save|store)\b(.*)$", low)
|
| 144 |
+
if m:
|
| 145 |
+
rest = m.group(1)
|
| 146 |
+
folder = None
|
| 147 |
+
for name, path in KNOWN_FOLDERS.items():
|
| 148 |
+
if name in rest:
|
| 149 |
+
folder = path
|
| 150 |
+
break
|
| 151 |
+
fname = None
|
| 152 |
+
fm = re.search(r"\bas\s+([\w\-. ]+)$", rest)
|
| 153 |
+
if fm:
|
| 154 |
+
fname = fm.group(1).strip()
|
| 155 |
+
steps.append(Step("save", rest.strip(), folder, fname))
|
| 156 |
+
continue
|
| 157 |
+
|
| 158 |
+
# An unrecognised clause makes the whole task unsafe to guess at.
|
| 159 |
+
return []
|
| 160 |
+
return steps
|
| 161 |
+
|
| 162 |
+
|
| 163 |
+
# --------------------------------------------------------------------------- #
|
| 164 |
+
# execution
|
| 165 |
+
# --------------------------------------------------------------------------- #
|
| 166 |
+
def _foreground_title() -> str:
|
| 167 |
+
if win32gui is None:
|
| 168 |
+
return ""
|
| 169 |
+
try:
|
| 170 |
+
return win32gui.GetWindowText(win32gui.GetForegroundWindow()) or ""
|
| 171 |
+
except Exception:
|
| 172 |
+
return ""
|
| 173 |
+
|
| 174 |
+
|
| 175 |
+
def _wait_for_window(fragment: str, timeout: float = 12.0) -> bool:
|
| 176 |
+
"""Wait until the app we launched is actually in front.
|
| 177 |
+
|
| 178 |
+
Typing before the window exists sends the keystrokes to whatever WAS in
|
| 179 |
+
front — which is how a naive version of this ends up typing into the user's
|
| 180 |
+
browser or, worse, a terminal.
|
| 181 |
+
"""
|
| 182 |
+
deadline = time.time() + timeout
|
| 183 |
+
fragment = fragment.lower()
|
| 184 |
+
while time.time() < deadline:
|
| 185 |
+
if fragment in _foreground_title().lower():
|
| 186 |
+
return True
|
| 187 |
+
time.sleep(0.35)
|
| 188 |
+
return False
|
| 189 |
+
|
| 190 |
+
|
| 191 |
+
SAVE_DIALOG_TITLES = ("save as", "save", "guardar", "enregistrer")
|
| 192 |
+
|
| 193 |
+
|
| 194 |
+
def _wait_for_save_dialog(timeout: float = 8.0) -> bool:
|
| 195 |
+
"""Block until a Save dialog owns the foreground window.
|
| 196 |
+
|
| 197 |
+
The editor's own title (for example "Untitled - Notepad") is explicitly NOT
|
| 198 |
+
a match: seeing it means the dialog has not opened yet and anything typed
|
| 199 |
+
would land in the document.
|
| 200 |
+
"""
|
| 201 |
+
deadline = time.time() + timeout
|
| 202 |
+
while time.time() < deadline:
|
| 203 |
+
title = _foreground_title().lower()
|
| 204 |
+
if title and any(t in title for t in SAVE_DIALOG_TITLES):
|
| 205 |
+
return True
|
| 206 |
+
time.sleep(0.25)
|
| 207 |
+
return False
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def _type_text(text: str) -> bool:
|
| 211 |
+
if keyboard is None:
|
| 212 |
+
return False
|
| 213 |
+
try:
|
| 214 |
+
# write() sends unicode directly rather than emulating scan codes, so
|
| 215 |
+
# punctuation and layout differences do not corrupt the text.
|
| 216 |
+
keyboard.write(text, delay=0.01)
|
| 217 |
+
return True
|
| 218 |
+
except Exception:
|
| 219 |
+
return False
|
| 220 |
+
|
| 221 |
+
|
| 222 |
+
def run_task(text: str, speak=None) -> TaskResult:
|
| 223 |
+
"""Execute a parsed multi-step task. Never reports a step it did not do."""
|
| 224 |
+
# Speak EVERY message as it happens, not in a batch at the end.
|
| 225 |
+
#
|
| 226 |
+
# The first version collected messages and only spoke them on the success
|
| 227 |
+
# path, so every failure returned with an empty `spoken` list and the caller
|
| 228 |
+
# fell back to "I don't have a handler for that yet" — hiding the actual
|
| 229 |
+
# reason the task stopped. A progress report that only appears when nothing
|
| 230 |
+
# went wrong is worse than none.
|
| 231 |
+
def say(msg: str) -> None:
|
| 232 |
+
if speak and msg:
|
| 233 |
+
speak(msg)
|
| 234 |
+
|
| 235 |
+
steps = parse_task(text)
|
| 236 |
+
if not steps:
|
| 237 |
+
return TaskResult(False, 0, ["I couldn't turn that into steps I can run."])
|
| 238 |
+
if keyboard is None or win32gui is None:
|
| 239 |
+
return TaskResult(False, 0,
|
| 240 |
+
["Desktop automation isn't available on this machine."])
|
| 241 |
+
|
| 242 |
+
messages: list[str] = []
|
| 243 |
+
def note(msg: str) -> None:
|
| 244 |
+
messages.append(msg)
|
| 245 |
+
say(msg)
|
| 246 |
+
done = 0
|
| 247 |
+
opened_app: str | None = None
|
| 248 |
+
|
| 249 |
+
for step in steps:
|
| 250 |
+
if step.kind == "open":
|
| 251 |
+
exe, title_fragment = KNOWN_APPS.get(step.value, (step.value, step.value))
|
| 252 |
+
try:
|
| 253 |
+
subprocess.Popen(
|
| 254 |
+
["cmd.exe", "/c", "start", "", exe],
|
| 255 |
+
stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
|
| 256 |
+
creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
|
| 257 |
+
except Exception as exc:
|
| 258 |
+
note(f"Couldn't open {step.value}: {exc}")
|
| 259 |
+
return TaskResult(False, done, messages)
|
| 260 |
+
# NEVER TYPE INTO A DOCUMENT THAT ALREADY HAS CONTENT.
|
| 261 |
+
#
|
| 262 |
+
# Windows 11 Notepad restores its previous session, so launching it
|
| 263 |
+
# can bring back the user's OWN unsaved work — and this code then
|
| 264 |
+
# typed into that document and pressed Ctrl+S, which saved silently
|
| 265 |
+
# because the tab already had a path. Verified happening: a tab
|
| 266 |
+
# holding a personal document was modified by a test run.
|
| 267 |
+
#
|
| 268 |
+
# Ctrl+N forces a brand-new empty document first. An editor that does
|
| 269 |
+
# not support it simply ignores the keystroke, so this is safe to
|
| 270 |
+
# send unconditionally — and infinitely safer than assuming a fresh
|
| 271 |
+
# window.
|
| 272 |
+
if _wait_for_window(title_fragment) and step.value in ("notepad", "wordpad", "word"):
|
| 273 |
+
try:
|
| 274 |
+
keyboard.send("ctrl+n")
|
| 275 |
+
time.sleep(1.0)
|
| 276 |
+
except Exception:
|
| 277 |
+
pass
|
| 278 |
+
|
| 279 |
+
if not _wait_for_window(title_fragment):
|
| 280 |
+
messages.append(
|
| 281 |
+
f"Opened {step.value}, but its window never came to the front — "
|
| 282 |
+
"stopping so I don't type into the wrong place.")
|
| 283 |
+
return TaskResult(False, done, messages)
|
| 284 |
+
opened_app = step.value
|
| 285 |
+
note(f"Opened {step.value}.")
|
| 286 |
+
done += 1
|
| 287 |
+
|
| 288 |
+
elif step.kind == "type":
|
| 289 |
+
if opened_app is None:
|
| 290 |
+
note("Nothing is open to type into.")
|
| 291 |
+
return TaskResult(False, done, messages)
|
| 292 |
+
time.sleep(0.4)
|
| 293 |
+
if not _type_text(step.value):
|
| 294 |
+
note("Couldn't send the text.")
|
| 295 |
+
return TaskResult(False, done, messages)
|
| 296 |
+
note(f'Typed "{step.value}".')
|
| 297 |
+
done += 1
|
| 298 |
+
|
| 299 |
+
elif step.kind == "save":
|
| 300 |
+
folder = step.folder or (Path.home() / "Desktop")
|
| 301 |
+
name = step.filename or f"jarvis-{time.strftime('%H%M%S')}.txt"
|
| 302 |
+
if not os.path.splitext(name)[1]:
|
| 303 |
+
name += ".txt"
|
| 304 |
+
target = folder / name
|
| 305 |
+
try:
|
| 306 |
+
folder.mkdir(parents=True, exist_ok=True)
|
| 307 |
+
except Exception:
|
| 308 |
+
pass
|
| 309 |
+
try:
|
| 310 |
+
keyboard.send("ctrl+s")
|
| 311 |
+
# WAIT FOR THE DIALOG, don't guess at a sleep.
|
| 312 |
+
#
|
| 313 |
+
# A fixed 1.2s worked when run from an interactive console and
|
| 314 |
+
# failed from the relay's background process, where the dialog
|
| 315 |
+
# takes longer to paint. The path was then typed into the
|
| 316 |
+
# DOCUMENT instead of the filename box and Enter just added a
|
| 317 |
+
# newline — the file never existed, and the only clue was a
|
| 318 |
+
# message the caller never saw. Same rule as the app window
|
| 319 |
+
# above: prove the target has focus before sending keys to it.
|
| 320 |
+
if not _wait_for_save_dialog():
|
| 321 |
+
note("The save dialog never appeared, so nothing was typed "
|
| 322 |
+
"into it — your text is still in the editor, unsaved.")
|
| 323 |
+
return TaskResult(False, done, messages)
|
| 324 |
+
# The Save dialog focuses its filename box; a full path in that
|
| 325 |
+
# box is honoured by the common dialog, so no navigation is
|
| 326 |
+
# needed and the file lands exactly where asked.
|
| 327 |
+
if not _type_text(str(target)):
|
| 328 |
+
note("Couldn't fill in the save dialog.")
|
| 329 |
+
return TaskResult(False, done, messages)
|
| 330 |
+
time.sleep(0.4)
|
| 331 |
+
keyboard.send("enter")
|
| 332 |
+
# Give the write a moment, then poll rather than assume.
|
| 333 |
+
for _ in range(20):
|
| 334 |
+
if target.exists():
|
| 335 |
+
break
|
| 336 |
+
time.sleep(0.25)
|
| 337 |
+
except Exception as exc:
|
| 338 |
+
note(f"Save failed: {exc}")
|
| 339 |
+
return TaskResult(False, done, messages)
|
| 340 |
+
|
| 341 |
+
# VERIFY. A save dialog can be refused, redirected or silently
|
| 342 |
+
# cancelled; the only proof is the file being on disk.
|
| 343 |
+
if target.exists():
|
| 344 |
+
note(f"Saved to {target}.")
|
| 345 |
+
done += 1
|
| 346 |
+
else:
|
| 347 |
+
# note(), not messages.append(): appended text is never spoken,
|
| 348 |
+
# so this exact reason was collected and thrown away — the caller
|
| 349 |
+
# saw an empty result and fell back to "no handler".
|
| 350 |
+
note(f"I pressed save, but {target} isn't on disk. Your text is "
|
| 351 |
+
"still open in the editor.")
|
| 352 |
+
return TaskResult(False, done, messages)
|
| 353 |
+
|
| 354 |
+
return TaskResult(True, done, messages)
|
| 355 |
+
|
| 356 |
+
|
| 357 |
+
def looks_like_task(text: str) -> bool:
|
| 358 |
+
"""Cheap gate for the dispatcher: at least two steps, one of them an action."""
|
| 359 |
+
steps = parse_task(text)
|
| 360 |
+
return len(steps) >= 2 and any(s.kind in ("type", "save") for s in steps)
|