Jarvis2345 commited on
Commit
e6cb1a4
·
verified ·
1 Parent(s): b30cb42

deploy(S4): Blender headless pipeline + WebAR client + backend fixes

Browse files
backend/events/omega_event_bus.py CHANGED
@@ -62,23 +62,29 @@ async def publish_omega_event(event: OmegaEvent):
62
  # 3. routes through existing §0.4 voice (stubbed for now if kokoro_engine not fully connected here,
63
  # but requested by user spec to synthesize_tts)
64
  if event_type_warrants_speech(event.event_type):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
65
  try:
66
- from backend.voice.engines.kokoro_engine import synthesize_kokoro as synthesize_tts
67
-
68
- persona = event.persona or "jarvis"
69
- # Context stub if ResponseContext does not exist natively
70
- class StubContext:
71
- type = "automation_status"
72
-
73
- await synthesize_tts(
74
- describe_event_for_speech(event),
75
- persona=persona,
76
- context=StubContext()
77
- )
78
- except ImportError as e:
79
- logging.warning(f"Could not import TTS engine for omega event speech: {e}")
80
  except Exception as e:
81
- logging.error(f"Error during TTS synthesis for omega event: {e}")
82
 
83
  # 4. Trigger vault auto-refresh
84
  try:
 
62
  # 3. routes through existing §0.4 voice (stubbed for now if kokoro_engine not fully connected here,
63
  # but requested by user spec to synthesize_tts)
64
  if event_type_warrants_speech(event.event_type):
65
+ # SPEAK the event, rather than synthesising it and dropping it.
66
+ #
67
+ # This called synthesize_kokoro directly and discarded the return value.
68
+ # synthesize_kokoro returns WAV bytes; it does not play anything and has
69
+ # no audio device. So every "feature implemented", "bug found",
70
+ # "game won" and "research alert" ran the full voice pipeline — model
71
+ # load, RVC conversion, the whole DSP chain — and then let the audio go
72
+ # out of scope unheard. Seconds of GPU work per event, for silence.
73
+ #
74
+ # It also ignored `persona`: the parameter was passed, but
75
+ # synthesize_kokoro takes **kwargs and only reads `profile`, so a FRIDAY
76
+ # event would have been rendered in the JARVIS voice had anyone heard it.
77
+ #
78
+ # core.voice.speak() is the correct destination. It picks the voice from
79
+ # the active persona, and its single-owner IPC means this backend
80
+ # process hands the line to whichever process owns the speakers instead
81
+ # of loading a second copy of the models.
82
  try:
83
+ from core.voice import speak
84
+ speak(describe_event_for_speech(event), event="omega_event",
85
+ cooldown_s=4.0)
 
 
 
 
 
 
 
 
 
 
 
86
  except Exception as e:
87
+ logging.error(f"Error speaking omega event: {e}")
88
 
89
  # 4. Trigger vault auto-refresh
90
  try:
backend/main.py CHANGED
@@ -113,6 +113,8 @@ from backend.routes.easter_egg_routes import router as easter_egg_router
113
  from backend.routes.omega_routes import router as omega_router
114
  from backend.routes.persona_routes import router as persona_router
115
  from backend.routes.internet_routes import router as internet_router
 
 
116
  from backend.routes.sentinel_routes import router as sentinel_router
117
  from backend.routers.gaming_routes import router as gaming_router
118
  from backend.routes.model_proxy_routes import router as model_proxy_router # AR-FIX-SESSION
@@ -206,6 +208,17 @@ app.include_router(distribution_router, prefix="")
206
  app.include_router(family_device_router, prefix="")
207
  app.include_router(persona_router, prefix="/persona", dependencies=[Depends(verify_token)])
208
  app.include_router(internet_router, prefix="/internet", dependencies=[Depends(verify_token)])
 
 
 
 
 
 
 
 
 
 
 
209
  app.include_router(sentinel_router, prefix="/sentinel", dependencies=[Depends(verify_token)])
210
  app.include_router(gaming_router, prefix="", dependencies=[Depends(verify_token)]) # gaming routes already have /gaming prefix
211
  app.include_router(model_proxy_router, prefix="/api") # AR-FIX-SESSION: model provider proxy stubs (no auth — WebAR calls these from browser)
@@ -690,7 +703,24 @@ def main():
690
  asyncio.run(run_all_migrations(db_path))
691
 
692
  cloud_env = os.environ.get("CLOUD_ENV", "false").lower() == "true"
693
- host = "0.0.0.0" if cloud_env else "127.0.0.1"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
694
  base_port = int(os.environ.get("PORT", "7860")) if cloud_env else int(os.environ.get("JARVIS_PORT", "7474"))
695
 
696
  for port in range(base_port, base_port + 11):
 
113
  from backend.routes.omega_routes import router as omega_router
114
  from backend.routes.persona_routes import router as persona_router
115
  from backend.routes.internet_routes import router as internet_router
116
+ from backend.routes.token_task_routes import router as token_task_router
117
+ from backend.routes.engine_routes import router as engine_router
118
  from backend.routes.sentinel_routes import router as sentinel_router
119
  from backend.routers.gaming_routes import router as gaming_router
120
  from backend.routes.model_proxy_routes import router as model_proxy_router # AR-FIX-SESSION
 
208
  app.include_router(family_device_router, prefix="")
209
  app.include_router(persona_router, prefix="/persona", dependencies=[Depends(verify_token)])
210
  app.include_router(internet_router, prefix="/internet", dependencies=[Depends(verify_token)])
211
+ # The token manager, reachable by the phone. Lets the handset use the full
212
+ # 37-key domain routing without carrying a single key itself.
213
+ app.include_router(token_task_router, prefix="/tokens", dependencies=[Depends(verify_token)])
214
+ # The full engine — unlimited tasks, intent, initiative, deferred orders,
215
+ # silence. One implementation, called by the phone rather than duplicated in
216
+ # Kotlin, so the handset can never drift from how JARVIS actually behaves.
217
+ #
218
+ # /engine, NOT /omega: omega_routes.py is the long-standing Autonomous Research
219
+ # and Enhancement API (research, propose, approve, upgrade) and already owns
220
+ # that prefix via its own APIRouter(prefix="/omega").
221
+ app.include_router(engine_router, prefix="/engine", dependencies=[Depends(verify_token)])
222
  app.include_router(sentinel_router, prefix="/sentinel", dependencies=[Depends(verify_token)])
223
  app.include_router(gaming_router, prefix="", dependencies=[Depends(verify_token)]) # gaming routes already have /gaming prefix
224
  app.include_router(model_proxy_router, prefix="/api") # AR-FIX-SESSION: model provider proxy stubs (no auth — WebAR calls these from browser)
 
703
  asyncio.run(run_all_migrations(db_path))
704
 
705
  cloud_env = os.environ.get("CLOUD_ENV", "false").lower() == "true"
706
+
707
+ # Bind the LAN, not just loopback.
708
+ #
709
+ # This was "0.0.0.0" only in cloud mode and "127.0.0.1" everywhere else, so
710
+ # on the desktop the bridge answered nothing but the machine it ran on. The
711
+ # phone's whole LAN path — the `http://<pc>:7474/api/...` half of the dual
712
+ # -path contract the Guardian app is built around — could never connect, and
713
+ # the JARVIS voice in particular cannot come from anywhere else: the cloud
714
+ # Space carries no RVC, so the PC is the only host that can render it.
715
+ #
716
+ # This is not an authentication hole. auth_middleware waives the token only
717
+ # for `request.client.host in ("127.0.0.1", "localhost", "::1")`; every LAN
718
+ # request must present a bearer token that passes verify_master_token, and
719
+ # that check is unchanged. What changes is only whether the socket is
720
+ # reachable at all.
721
+ #
722
+ # JARVIS_BIND_HOST forces it back to loopback on an untrusted network.
723
+ host = os.environ.get("JARVIS_BIND_HOST") or "0.0.0.0"
724
  base_port = int(os.environ.get("PORT", "7860")) if cloud_env else int(os.environ.get("JARVIS_PORT", "7474"))
725
 
726
  for port in range(base_port, base_port + 11):
backend/routes/engine_routes.py ADDED
@@ -0,0 +1,310 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """backend/routes/engine_routes.py — the whole engine, reachable from anywhere.
2
+
3
+ WHY A ROUTE AND NOT A KOTLIN PORT
4
+ ---------------------------------
5
+ Everything built this session is Python: modules/omega_executor.py plans and
6
+ runs unlimited tasks (writing its own code when nothing fits),
7
+ modules/intent.py understands what was meant rather than matching words,
8
+ modules/deferred_intent.py deliberates irreversible orders, and
9
+ modules/initiative.py acts unprompted.
10
+
11
+ Re-implementing those four in Kotlin would give the phone a SECOND definition
12
+ of how JARVIS thinks and acts — which is exactly the mistake
13
+ MobileJarvisEngine.kt already made with the persona text, and exactly how
14
+ core/brain.py drifted into telling JARVIS he was FRIDAY on every turn. Two
15
+ copies of a brain diverge.
16
+
17
+ So the phone calls this. One engine, one behaviour, and anything added to the
18
+ Python is instantly available on the handset with no rebuild.
19
+
20
+ WHERE THE WORK HAPPENS
21
+ ----------------------
22
+ On the machine running this backend. That is the point: "shut down the PC",
23
+ "finish that download", "close every window" are PC actions, and the phone is
24
+ the remote. Requests that are really about the handset itself (torch, phone
25
+ volume) still belong to the on-device engine, and this does not replace that.
26
+ """
27
+
28
+ from typing import Optional
29
+
30
+ from fastapi import APIRouter, HTTPException
31
+ from pydantic import BaseModel
32
+
33
+ router = APIRouter()
34
+
35
+
36
+ class ExecuteRequest(BaseModel):
37
+ request: str
38
+ context: str = ""
39
+ # Off by default over the network: an irreversible action asked for from a
40
+ # phone in a pocket deserves a deliberate opt-in, not a default.
41
+ allow_destructive: bool = False
42
+
43
+
44
+ class PlanRequest(BaseModel):
45
+ request: str
46
+ context: str = ""
47
+
48
+
49
+ class IntentRequest(BaseModel):
50
+ text: str
51
+
52
+
53
+ @router.post("/execute")
54
+ async def omega_execute(req: ExecuteRequest) -> dict:
55
+ """Do whatever was asked — the full unlimited engine.
56
+
57
+ Chains, waits, real commands, and model-authored code for anything with no
58
+ existing handler. Identical to what the desktop voice path runs.
59
+ """
60
+ text = (req.request or "").strip()
61
+ if not text:
62
+ raise HTTPException(status_code=400, detail="request is required")
63
+ try:
64
+ from modules.omega_executor import execute
65
+ spoken: list[str] = []
66
+ res = execute(
67
+ text,
68
+ speak=spoken.append,
69
+ context=req.context or "",
70
+ allow_destructive=bool(req.allow_destructive),
71
+ )
72
+ return {
73
+ "status": "ok",
74
+ "ok": res.ok,
75
+ "spoken": res.spoken,
76
+ "said": spoken,
77
+ "cancelled": res.cancelled,
78
+ "steps": [
79
+ {"index": s.index, "kind": s.kind, "summary": s.summary,
80
+ "ok": s.ok, "output": s.output[:1200], "error": s.error[:600]}
81
+ for s in res.steps
82
+ ],
83
+ }
84
+ except Exception as e:
85
+ raise HTTPException(status_code=500, detail=str(e))
86
+
87
+
88
+ @router.post("/plan")
89
+ async def omega_plan(req: PlanRequest) -> dict:
90
+ """Plan without doing. Lets the phone show what WOULD happen first."""
91
+ text = (req.request or "").strip()
92
+ if not text:
93
+ raise HTTPException(status_code=400, detail="request is required")
94
+ try:
95
+ from modules.omega_executor import plan_request, is_destructive
96
+ plan = plan_request(text, req.context or "")
97
+ steps = plan.get("steps") or []
98
+ return {
99
+ "status": "ok",
100
+ "say": plan.get("say", ""),
101
+ "degraded": bool(plan.get("degraded")),
102
+ "step_count": len(steps),
103
+ "steps": steps,
104
+ "has_irreversible": any(
105
+ is_destructive(str(s.get("cmd") or s.get("code") or "")
106
+ + " " + str(s.get("name") or ""))
107
+ for s in steps
108
+ ),
109
+ }
110
+ except Exception as e:
111
+ raise HTTPException(status_code=500, detail=str(e))
112
+
113
+
114
+ @router.post("/intent")
115
+ async def omega_intent(req: IntentRequest) -> dict:
116
+ """What did they MEAN? act / converse / wake / mute / unmute / cancel."""
117
+ try:
118
+ from modules.intent import classify
119
+ got = classify(req.text or "")
120
+ return {
121
+ "status": "ok",
122
+ "intent": got.kind,
123
+ "confidence": got.confidence,
124
+ "payload": got.payload,
125
+ "source": got.source,
126
+ }
127
+ except Exception as e:
128
+ raise HTTPException(status_code=500, detail=str(e))
129
+
130
+
131
+ @router.get("/repairs")
132
+ async def list_repairs() -> dict:
133
+ """Commands that had to be corrected at runtime.
134
+
135
+ The immediate repair unblocks the operator; this is the evidence trail for
136
+ the lasting fix. A command that needs correcting every single run is a bug
137
+ that self-repair is otherwise hiding very effectively.
138
+ """
139
+ try:
140
+ from modules.omega_executor import pending_repairs
141
+ items = pending_repairs()
142
+ return {"status": "ok", "count": len(items), "repairs": items[-25:]}
143
+ except Exception as e:
144
+ raise HTTPException(status_code=500, detail=str(e))
145
+
146
+
147
+ @router.get("/depth_model_url")
148
+ async def depth_model_url() -> dict:
149
+ """Where the phone should fetch the monocular-depth weights.
150
+
151
+ Served rather than hardcoded in the APK because a guessed URL that fails
152
+ leaves 2D->3D silently off forever. Three candidates were tested live and
153
+ all three failed (401 on a private path, 404 on two public mirrors), which
154
+ is exactly why this is configuration and not a constant.
155
+
156
+ Set OMEGA_DEPTH_MODEL_URL to a reachable .onnx and every handset picks it
157
+ up with no rebuild. Until then this returns empty and the phone reports
158
+ "no model URL configured" instead of pretending to be a 3D viewer.
159
+ """
160
+ import os
161
+
162
+ # Depth Anything V2 small, int8. Chosen because:
163
+ # * the repo and file were CONFIRMED to exist via the HF API
164
+ # (onnx-community/depth-anything-v2-small -> onnx/model_int8.onnx);
165
+ # * int8 keeps it small enough to download and run on a handset;
166
+ # * V2 small is markedly better than MiDaS small on indoor scenes, which
167
+ # is what a photo or video in a virtual theatre usually is.
168
+ #
169
+ # NOT verified by downloading: this environment cannot follow Hugging Face's
170
+ # CDN redirect, so the bytes were never fetched here. The path is real; the
171
+ # first phone to run it is the real test.
172
+ DEFAULT = ("https://huggingface.co/onnx-community/depth-anything-v2-small"
173
+ "/resolve/main/onnx/model_int8.onnx")
174
+
175
+ url = (os.environ.get("OMEGA_DEPTH_MODEL_URL") or DEFAULT).strip()
176
+
177
+ # Input size travels WITH the url. Getting these out of step gives a model
178
+ # that loads happily and returns nonsense depth, which shows up as a
179
+ # headache rather than an error — far worse than a clean failure.
180
+ try:
181
+ size = int(os.environ.get("OMEGA_DEPTH_INPUT_SIZE") or 0)
182
+ except ValueError:
183
+ size = 0
184
+ if not size:
185
+ size = 256 if "midas" in url.lower() else 518
186
+
187
+ return {
188
+ "status": "ok",
189
+ "url": url,
190
+ "input_size": size,
191
+ "configured": True,
192
+ "is_default": url == DEFAULT,
193
+ "note": ("Default Depth Anything V2 small (int8). Override with "
194
+ "OMEGA_DEPTH_MODEL_URL, and set OMEGA_DEPTH_INPUT_SIZE to "
195
+ "match that model's expected input."),
196
+ }
197
+
198
+
199
+ @router.get("/capabilities")
200
+ async def omega_capabilities() -> dict:
201
+ """Every power currently granted. Honest about what it can actually do."""
202
+ try:
203
+ from modules.omega_executor import capability_catalog
204
+ caps = capability_catalog()
205
+ return {"status": "ok", "count": len(caps), "capabilities": caps}
206
+ except Exception as e:
207
+ raise HTTPException(status_code=500, detail=str(e))
208
+
209
+
210
+ # ── Unprompted action, from the phone's point of view ───────────────────────
211
+ @router.get("/initiative")
212
+ async def initiative_state() -> dict:
213
+ """What he noticed, and anything he did while nobody was watching."""
214
+ try:
215
+ from modules.initiative import observe, pending_report, format_report, is_running
216
+ return {
217
+ "status": "ok",
218
+ "running": is_running(),
219
+ "situation": observe(),
220
+ "pending_report": pending_report(),
221
+ "spoken_report": format_report(),
222
+ }
223
+ except Exception as e:
224
+ raise HTTPException(status_code=500, detail=str(e))
225
+
226
+
227
+ @router.post("/initiative/tick")
228
+ async def initiative_tick() -> dict:
229
+ """Look now rather than waiting for the timer."""
230
+ try:
231
+ from modules.initiative import tick
232
+ spoken: list[str] = []
233
+ v = tick(speak=spoken.append)
234
+ if v is None:
235
+ return {"status": "ok", "silenced": True}
236
+ return {
237
+ "status": "ok",
238
+ "act": v.act, "say": v.say, "action": v.action,
239
+ "urgency": v.urgency, "topic": v.topic, "said": spoken,
240
+ }
241
+ except Exception as e:
242
+ raise HTTPException(status_code=500, detail=str(e))
243
+
244
+
245
+ @router.post("/initiative/acknowledge")
246
+ async def initiative_ack() -> dict:
247
+ """The operator has seen the report; stop offering it."""
248
+ try:
249
+ from modules.initiative import mark_reported, note_user_contact
250
+ note_user_contact()
251
+ mark_reported()
252
+ return {"status": "ok"}
253
+ except Exception as e:
254
+ raise HTTPException(status_code=500, detail=str(e))
255
+
256
+
257
+ # ── Held irreversible orders ────────────────────────────────────────────────
258
+ @router.get("/deferred")
259
+ async def deferred_list() -> dict:
260
+ """Orders held back, why, and when they will be reconsidered."""
261
+ try:
262
+ from modules.deferred_intent import pending
263
+ return {"status": "ok", "orders": pending()}
264
+ except Exception as e:
265
+ raise HTTPException(status_code=500, detail=str(e))
266
+
267
+
268
+ class WithdrawRequest(BaseModel):
269
+ order: Optional[str] = None
270
+
271
+
272
+ @router.post("/deferred/withdraw")
273
+ async def deferred_withdraw(req: WithdrawRequest) -> dict:
274
+ """Changed their mind. Drops one order, or all of them."""
275
+ try:
276
+ from modules.deferred_intent import withdraw
277
+ withdraw(req.order)
278
+ return {"status": "ok"}
279
+ except Exception as e:
280
+ raise HTTPException(status_code=500, detail=str(e))
281
+
282
+
283
+ # ── Silence ─────────────────────────────────────────────────────────────────
284
+ class SilenceRequest(BaseModel):
285
+ silent: bool
286
+ reason: str = ""
287
+
288
+
289
+ @router.post("/silence")
290
+ async def set_silence(req: SilenceRequest) -> dict:
291
+ """Total mute, from the phone. No timeout — released only explicitly."""
292
+ try:
293
+ from modules.alert_manager import silence_all, unsilence_all, is_silenced
294
+ if req.silent:
295
+ silence_all(req.reason or "asked from phone")
296
+ else:
297
+ unsilence_all()
298
+ return {"status": "ok", "silenced": is_silenced()}
299
+ except Exception as e:
300
+ raise HTTPException(status_code=500, detail=str(e))
301
+
302
+
303
+ @router.get("/silence")
304
+ async def get_silence() -> dict:
305
+ try:
306
+ from modules.alert_manager import is_silenced, silence_info
307
+ return {"status": "ok", "silenced": is_silenced(), "info": silence_info()}
308
+ except Exception as e:
309
+ raise HTTPException(status_code=500, detail=str(e))
310
+
backend/routes/mobile_bridge_routes.py CHANGED
@@ -209,6 +209,15 @@ async def link_status() -> dict[str, Any]:
209
  "detail": detail,
210
  "queued_commands": queued,
211
  "pc_status_age_seconds": age,
 
 
 
 
 
 
 
 
 
212
  "pc_status": status,
213
  }
214
 
@@ -357,6 +366,351 @@ def _gemini_envelope(text: str) -> dict[str, Any]:
357
  }
358
 
359
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
360
  @router.post("/chat")
361
  async def chat(request: Request) -> dict[str, Any]:
362
  """Synchronous LLM turn for the mobile app.
@@ -372,6 +726,15 @@ async def chat(request: Request) -> dict[str, Any]:
372
  This runs the prompt synchronously through the token manager's full
373
  Gemini→NVIDIA fallback chain and returns the Gemini envelope, so both callers
374
  work with no APK change.
 
 
 
 
 
 
 
 
 
375
  """
376
  body = await _json_body(request)
377
  text = str(body.get("text") or "").strip()
@@ -381,9 +744,34 @@ async def chat(request: Request) -> dict[str, Any]:
381
  persona = "jarvis"
382
  if not text:
383
  return _gemini_envelope("Standing by.")
 
 
 
384
  try:
385
  from backend.services.token_manager import gemini_call_with_checkpoint
386
- reply = await gemini_call_with_checkpoint(text, task_type="general", persona=persona)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
387
  return _gemini_envelope(reply or "")
388
  except Exception as exc:
389
  log.warning("mobile /api/chat LLM call failed: %s", exc)
 
209
  "detail": detail,
210
  "queued_commands": queued,
211
  "pc_status_age_seconds": age,
212
+ # Where the phone can reach the PC directly for VOICE.
213
+ #
214
+ # Commands need no address — the cloud relays them — but the voice
215
+ # cannot work that way: this Space has no RVC and no model weights, so
216
+ # only the PC can render it. Rather than make the user type an IP, the
217
+ # desktop advertises its own reachable addresses in its periodic status
218
+ # and they are handed straight through here, on the call the app already
219
+ # polls. Tailscale first, then LAN. Empty until a PC has reported.
220
+ "voice_hosts": status.get("voice_hosts") or [],
221
  "pc_status": status,
222
  }
223
 
 
366
  }
367
 
368
 
369
+ async def _measure_live_state(context: dict[str, Any]) -> dict[str, Any]:
370
+ """The facts of this moment, measured rather than assumed.
371
+
372
+ Asked "is the PC connected?", the assistant used to answer from nothing at
373
+ all — the chat path sent the characterisation and the user's sentence and no
374
+ system state whatsoever — so it produced a confident "connected, sir" about a
375
+ machine that was switched off. Everything below is read live; anything that
376
+ cannot be read is reported as unknown rather than omitted, because a stated
377
+ unknown is far harder to confabulate over than a silent gap.
378
+ """
379
+ state: dict[str, Any] = {}
380
+
381
+ # PC link — the authoritative answer is whether a desktop is holding an
382
+ # outbound socket right now, not whether one was ever paired.
383
+ try:
384
+ from backend.ws.agent_ws import ws_manager
385
+ pcs = list(getattr(ws_manager, "pc_connections", []) or [])
386
+ state["PC connected right now"] = "yes" if pcs else "no - the PC is offline"
387
+ state["PC sockets held"] = len(pcs)
388
+ except Exception as exc:
389
+ log.debug("live state: pc link unreadable: %s", exc)
390
+ state["PC connected right now"] = "unknown - could not read the link"
391
+
392
+ try:
393
+ import datetime
394
+ state["current time"] = datetime.datetime.now().strftime("%Y-%m-%d %H:%M %Z").strip()
395
+ except Exception:
396
+ pass
397
+
398
+ for key, label in (
399
+ ("device_name", "operator device"),
400
+ ("surface", "which surface this came from"),
401
+ ("location", "operator location"),
402
+ ):
403
+ val = context.get(key)
404
+ if val:
405
+ state[label] = str(val)[:120]
406
+ state.setdefault("which surface this came from", "the phone app")
407
+
408
+ # The handset's own senses — where the operator is, what they are doing, who
409
+ # is around, whether speech would be overheard. SituationSensor.kt gathers
410
+ # these; anything it could not read is simply absent, never invented.
411
+ #
412
+ # This is the difference between an assistant that adapts to the situations
413
+ # somebody thought to enumerate and one that adapts to the situation actually
414
+ # occurring. The readings are raw on purpose: "ringer: vibrate, calendar
415
+ # event: standup, headphones: no" is evidence he reasons over, not a label
416
+ # like "in a meeting" decided for him by a table on the phone.
417
+ senses = context.get("senses")
418
+ if isinstance(senses, dict):
419
+ for key, val in senses.items():
420
+ if val not in (None, ""):
421
+ state[str(key)[:60]] = str(val)[:160]
422
+
423
+ # Held irreversible orders and unprompted work, when this process has them.
424
+ try:
425
+ from modules.deferred_intent import pending as _deferred_pending
426
+ held = _deferred_pending() or []
427
+ if held:
428
+ state["orders held pending deliberation"] = "; ".join(
429
+ str(o.get("order", ""))[:80] for o in held[:4]
430
+ )
431
+ except Exception:
432
+ pass
433
+ try:
434
+ from modules.initiative import pending_report as _init_pending
435
+ done = _init_pending() or []
436
+ if done:
437
+ state["things done unprompted, not yet reported"] = len(done)
438
+ except Exception:
439
+ pass
440
+
441
+ return state
442
+
443
+
444
+ # Separate budgets, because they used to share one.
445
+ #
446
+ # Three rounds total meant a single PC run plus a single search exhausted the
447
+ # turn, and the operator got the canned "I couldn't get to a straight answer"
448
+ # even though nothing had failed. Real research is: search, read what came back,
449
+ # search again for the specific thing. That is three searches minimum before a
450
+ # word is said.
451
+ _MAX_ROUNDS = 7 # hard stop on the whole turn
452
+ _MAX_SEARCHES = 4 # web lookups per turn
453
+ _MAX_PC_RUNS = 2 # desktop executions per turn
454
+
455
+
456
+ async def _run_on_pc(task: str) -> str:
457
+ """Run an arbitrary task on the operator's desktop and report what happened.
458
+
459
+ Two routes, in order of directness. On the Space there is no desktop to run
460
+ on, so the work goes down the socket the PC holds open (the same path
461
+ /max_autonomy/task uses). Running in-process is correct when this backend IS
462
+ the desktop, which is how the exe serves it.
463
+
464
+ Irreversible steps are deliberated over and held with a cancel window inside
465
+ the executor itself, so nothing here needs to second-guess the request.
466
+ """
467
+ if not task:
468
+ return "No task given."
469
+
470
+ # In-process: this backend is the machine.
471
+ try:
472
+ from modules.omega_executor import execute as _execute
473
+ import asyncio as _asyncio
474
+ result = await _asyncio.to_thread(_execute, task)
475
+ steps = getattr(result, "steps", None) or []
476
+ lines = [f"{'ok' if getattr(s, 'ok', False) else 'FAILED'}: "
477
+ f"{getattr(s, 'summary', '')}"[:180] for s in steps[:12]]
478
+ spoken = getattr(result, "spoken", "") or ""
479
+ return (spoken + "\n" + "\n".join(lines)).strip() or "Completed with no output."
480
+ except ImportError:
481
+ pass
482
+ except Exception as exc:
483
+ log.warning("in-process executor failed: %s", exc)
484
+ return f"The engine errored: {str(exc)[:300]}"
485
+
486
+ # Cloud: relay to whichever desktop is holding a socket open.
487
+ try:
488
+ from backend.ws.agent_ws import ws_manager
489
+ if not getattr(ws_manager, "pc_connections", None):
490
+ return ("The PC is offline, so there was nothing to run it on. "
491
+ "It was not queued.")
492
+ outcome = await ws_manager.execute_on_pcs(task)
493
+ if not outcome.get("delivered"):
494
+ return "The PC did not accept the task."
495
+ if outcome.get("timed_out"):
496
+ return "The PC took the task and is still working on it."
497
+ said = outcome.get("spoken") or []
498
+ return "\n".join(str(s) for s in said)[:1500] or "Done, with no output reported."
499
+ except Exception as exc:
500
+ log.warning("PC relay failed: %s", exc)
501
+ return f"Could not reach the PC: {str(exc)[:200]}"
502
+
503
+ # Readings that describe the operator's circumstances rather than the machine's.
504
+ # Only these go in the preamble; battery and PC-link facts belong in LIVE STATE
505
+ # where they can be quoted, not in the line that shapes how he answers.
506
+ _CIRCUMSTANCE_KEYS = (
507
+ "local time", "part of day", "day", "roughly where", "moving",
508
+ "app in use", "screen", "in a calendar event right now", "event location",
509
+ "bluetooth connected", "ringer", "media playing",
510
+ "speaking aloud would be overheard", "wifi network",
511
+ )
512
+
513
+
514
+ def _situated(text: str, live: dict[str, Any]) -> str:
515
+ """Put the operator's circumstances immediately before their question.
516
+
517
+ LIVE STATE already carries these, at the end of a system prompt that runs to
518
+ a hundred and fifty lines of characterisation. That is enough for him to
519
+ QUOTE a reading when asked, and demonstrably not enough for him to ACT on
520
+ one: given 68 km/h and a car's bluetooth, he cheerfully began reciting a
521
+ multi-step server setup to a man at the wheel.
522
+
523
+ Attention follows recency far more than depth, so the same facts are
524
+ restated here, adjacent to the question, as the thing to think about first.
525
+ Same readings, no new claims — only a position that gets them used.
526
+ """
527
+ bits = [f"{k}: {v}" for k, v in live.items()
528
+ if k in _CIRCUMSTANCE_KEYS and v not in (None, "", "unknown")]
529
+ if not bits:
530
+ return text
531
+ return (
532
+ "RIGHT NOW, around the operator: " + "; ".join(bits) + ".\n"
533
+ "Work out what that means about what they are doing, what they can "
534
+ "actually use this second, and who can hear.\n"
535
+ "\n"
536
+ "DECIDE THE SHAPE OF YOUR ANSWER BEFORE YOU WRITE IT, NOT AFTER.\n"
537
+ "\n"
538
+ "If the circumstances mean they cannot use a long answer right now, then "
539
+ "the short answer IS the answer. Do not deliver the long one and then "
540
+ "remark that they cannot use it. Giving a man at the wheel four steps of "
541
+ "server configuration and closing with 'of course, you are driving' is "
542
+ "not judgement — it is the whole procedure plus an apology for it, and "
543
+ "he could not act on a word of it. What he needed was the one-line "
544
+ "answer and an offer to walk him through it when he is somewhere he can "
545
+ "type. Same for anything overheard: if it should not be said aloud here, "
546
+ "do not say it and then note that it was sensitive.\n"
547
+ "\n"
548
+ "Judge every time from the readings themselves — these are examples of "
549
+ "the reasoning, not a list of cases to match. Do not mention the "
550
+ "readings unless they are what was asked about.\n\n"
551
+ f"THEY SAID: {text}"
552
+ )
553
+
554
+
555
+ def _history_block(history: list[Any]) -> str:
556
+ """The last few turns, as plain transcript.
557
+
558
+ Without this the endpoint is amnesiac: every POST carries one utterance and
559
+ no antecedent, so "download it" has no *it*. The handset has kept a six-turn
560
+ window for a long time — it was simply never sent (see MobileJarvisEngine,
561
+ where the request carrying it was built and then dropped on the floor).
562
+
563
+ A pronoun with nothing to bind to is the single most common way this
564
+ assistant looked stupid, and it was never a reasoning failure.
565
+ """
566
+ rows = []
567
+ for turn in history[-8:]:
568
+ if isinstance(turn, dict):
569
+ role = str(turn.get("role") or "user").strip().lower()
570
+ said = str(turn.get("text") or "").strip()
571
+ elif isinstance(turn, (list, tuple)) and len(turn) == 2:
572
+ role, said = str(turn[0]).strip().lower(), str(turn[1]).strip()
573
+ else:
574
+ continue
575
+ if not said:
576
+ continue
577
+ who = "THEY" if role in ("user", "operator", "human") else "YOU"
578
+ rows.append(f"{who}: {said[:600]}")
579
+ if not rows:
580
+ return ""
581
+ return (
582
+ "THE CONVERSATION SO FAR (oldest first). Pronouns in the new message — "
583
+ "'it', 'that', 'her', 'the same one' — almost always refer to something "
584
+ "here. Resolve them from this before doing anything else:\n"
585
+ + "\n".join(rows) + "\n\n"
586
+ )
587
+
588
+
589
+ async def _run_lookup_loop(
590
+ text: str,
591
+ persona: str,
592
+ live: dict[str, Any],
593
+ history: list[Any] | None = None,
594
+ ) -> str:
595
+ """Ask, and honour a SEARCH: line by actually searching, then ask again.
596
+
597
+ This is what makes "who is <a real person>" answerable. Previously chat was a
598
+ bare model call with no way to reach the internet, so any question about
599
+ someone the model did not already know came back as "I can't find them" —
600
+ without a single lookup having been attempted. search_web already existed and
601
+ was wired to research and sentinel, never to conversation.
602
+
603
+ The decision to search is the model's, per turn, rather than a list of
604
+ question shapes that mean "search now" — a list would fail on the first shape
605
+ nobody thought of, which is the exact failure being fixed.
606
+ """
607
+ from backend.services.token_manager import gemini_call_with_checkpoint
608
+
609
+ # Everything learned this turn, oldest first, carried into every later round.
610
+ #
611
+ # This used to be a single `prompt` variable that each round OVERWROTE, so a
612
+ # second search silently erased the first one's results. Any request needing
613
+ # two steps — find the official page, then find the file on it — could not
614
+ # work: by the time he had the second answer he had lost the first. That is
615
+ # research reduced to one lucky query.
616
+ evidence: list[str] = []
617
+ asked = _situated(text, live)
618
+ if history:
619
+ asked = _history_block(history) + asked
620
+
621
+ def _compose() -> str:
622
+ if not evidence:
623
+ return asked
624
+ return (
625
+ "WHAT YOU HAVE ACTUALLY DONE AND FOUND SO FAR THIS TURN, in order:\n\n"
626
+ + "\n\n".join(evidence)
627
+ + "\n\nUse it. If it answers them, answer now and name what you found "
628
+ "— the real title, the real link, the real number. If it does not, "
629
+ "either search once more for the missing piece or tell them plainly "
630
+ "that you could not find it.\n\n"
631
+ "NEVER report something as done that you did not do, and never "
632
+ "describe a result you did not receive. 'I couldn't find it' is a "
633
+ "good answer. A confident invention is not.\n\n"
634
+ + asked
635
+ )
636
+
637
+ searches = 0
638
+ pc_runs = 0
639
+ for _ in range(_MAX_ROUNDS):
640
+ reply = (await gemini_call_with_checkpoint(
641
+ _compose(), task_type="general", persona=persona, live_state=live,
642
+ can_search=(searches < _MAX_SEARCHES), can_act=True, can_read=True,
643
+ ) or "").strip()
644
+
645
+ # READ is fulfilled on the PHONE, not here — only the handset can see its
646
+ # own screen, notifications, messages and clipboard. So a READ: reply is
647
+ # handed back verbatim; MobileJarvisEngine on the device performs the read
648
+ # and re-POSTs with the result. The cloud has no eyes on the phone and
649
+ # must not pretend to.
650
+ if reply.upper().startswith("READ:"):
651
+ return reply
652
+
653
+ # Hand the whole thing to the unlimited engine on the desktop.
654
+ #
655
+ # This is the escape hatch that makes "anything, including things nobody
656
+ # coded" true rather than aspirational: omega_executor plans a request
657
+ # into steps, runs shell, and writes fresh Python when no existing tool
658
+ # fits, repairing a failed step and retrying. The chat path had no idea
659
+ # it existed, so the assistant refused work his own machine could do.
660
+ if reply.upper().startswith("PC:"):
661
+ task = reply.split(":", 1)[1].strip()
662
+ if pc_runs >= _MAX_PC_RUNS:
663
+ evidence.append(
664
+ f"You wanted the desktop to: {task}\nYou have already used the "
665
+ "desktop as many times as this turn allows. Work with what you "
666
+ "have, or say what is still missing.")
667
+ continue
668
+ pc_runs += 1
669
+ outcome = await _run_on_pc(task)
670
+ evidence.append(f"YOU RAN ON THE DESKTOP: {task}\nWHAT HAPPENED:\n{outcome}")
671
+ continue
672
+
673
+ if not reply.upper().startswith("SEARCH:"):
674
+ return reply
675
+
676
+ query = reply.split(":", 1)[1].strip()
677
+ if not query:
678
+ return reply
679
+ searches += 1
680
+ log.info("chat: assistant requested a lookup: %s", query[:120])
681
+ try:
682
+ from backend.tools.web_search_tools import search_web
683
+ found = await search_web(query, num_results=5)
684
+ except Exception as exc:
685
+ log.warning("chat lookup failed: %s", exc)
686
+ found = {"error": str(exc)[:200]}
687
+
688
+ if found.get("error"):
689
+ block = f"It failed: {found['error']}. You did NOT get any results."
690
+ else:
691
+ rows = found.get("results") or []
692
+ if not rows:
693
+ block = "It returned nothing. You did NOT find anything."
694
+ else:
695
+ block = "\n".join(
696
+ f"- {r.get('title', '')}: {str(r.get('snippet') or r.get('body') or '')[:300]}"
697
+ f" ({r.get('url') or r.get('link') or r.get('href') or ''})"
698
+ for r in rows[:5]
699
+ )
700
+ evidence.append(f"YOU SEARCHED THE WEB FOR: {query}\nWHAT CAME BACK:\n{block}")
701
+
702
+ # Out of rounds. Rather than a canned apology, make one last call with
703
+ # everything gathered and no way to ask for more, so the operator gets the
704
+ # best answer the evidence supports instead of a shrug.
705
+ final = (await gemini_call_with_checkpoint(
706
+ _compose() + "\n\nYou are out of lookups. Answer them now from what you "
707
+ "have. If it is not enough, say exactly what you could not find.",
708
+ task_type="general", persona=persona, live_state=live,
709
+ can_search=False, can_act=True, can_read=False,
710
+ ) or "").strip()
711
+ return final or "I looked, but I couldn't get to a straight answer on that."
712
+
713
+
714
  @router.post("/chat")
715
  async def chat(request: Request) -> dict[str, Any]:
716
  """Synchronous LLM turn for the mobile app.
 
726
  This runs the prompt synchronously through the token manager's full
727
  Gemini→NVIDIA fallback chain and returns the Gemini envelope, so both callers
728
  work with no APK change.
729
+
730
+ THE PHONE'S BRAIN IS THIS FUNCTION. Everything the assistant is supposed to
731
+ have — situational judgement about how much to say, refusal to invent system
732
+ state, the ability to look something up — has to be here, because the desktop
733
+ brain in core/brain.py is a different process the handset never reaches. Work
734
+ put only into core/brain.py is, from the phone's point of view, not built.
735
+
736
+ The automation loop is exempt: it wants strict CLICK/INPUT/SCROLL/DONE tokens
737
+ and would be broken by conversational shaping or a lookup detour.
738
  """
739
  body = await _json_body(request)
740
  text = str(body.get("text") or "").strip()
 
744
  persona = "jarvis"
745
  if not text:
746
  return _gemini_envelope("Standing by.")
747
+
748
+ structured = bool(context.get("structured")) or str(context.get("mode") or "") == "automation"
749
+
750
  try:
751
  from backend.services.token_manager import gemini_call_with_checkpoint
752
+ if structured:
753
+ reply = await gemini_call_with_checkpoint(text, task_type="automation", persona=persona)
754
+ return _gemini_envelope(reply or "")
755
+
756
+ live = await _measure_live_state(context)
757
+ raw_history = context.get("history")
758
+ history = raw_history if isinstance(raw_history, list) else []
759
+ reply = await _run_lookup_loop(text, persona, live, history)
760
+
761
+ # Spoken aloud on the handset: strip any markdown that survived the
762
+ # instruction not to write it.
763
+ #
764
+ # An action reply is JSON the app parses and executes, not speech, so it
765
+ # is passed through untouched — running a formatting filter over a
766
+ # payload would corrupt the very thing that carries out the request.
767
+ stripped = reply.lstrip()
768
+ is_action = stripped.startswith("{") and '"actions"' in stripped
769
+ if not is_action:
770
+ try:
771
+ from modules.assistant_identity import to_spoken
772
+ reply = to_spoken(reply)
773
+ except Exception:
774
+ pass
775
  return _gemini_envelope(reply or "")
776
  except Exception as exc:
777
  log.warning("mobile /api/chat LLM call failed: %s", exc)
backend/routes/persona_routes.py CHANGED
@@ -52,13 +52,47 @@ async def get_persona():
52
 
53
 
54
  @router.get("/prompts")
55
- async def get_persona_prompts():
56
- """Return the full system prompts for both personas (sanitised for UI display)."""
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
  try:
58
- from modules.assistant_identity import JARVIS_PERSONALITY_PROMPT, FRIDAY_PERSONALITY_PROMPT
59
- return {
60
- "jarvis": JARVIS_PERSONALITY_PROMPT[:500] + "...",
61
- "friday": FRIDAY_PERSONALITY_PROMPT[:500] + "...",
62
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
63
  except Exception as e:
64
  raise HTTPException(status_code=500, detail=str(e))
 
52
 
53
 
54
  @router.get("/prompts")
55
+ async def get_persona_prompts(full: bool = False):
56
+ """The persona system prompts.
57
+
58
+ `full=true` returns them COMPLETE, including each persona's own tastes, and
59
+ that is the point of this route now rather than a nicety.
60
+
61
+ assistant_identity.py is meant to be the single source of truth for who the
62
+ assistant is. It is not, in practice: MobileJarvisEngine.kt carries its own
63
+ hardcoded Kotlin copy of the JARVIS characterisation, because an Android app
64
+ cannot import a Python module. Two copies of a personality drift — that is
65
+ exactly how core/brain.py ended up telling JARVIS he was FRIDAY on every
66
+ turn — and the phone's copy is the last one still able to drift.
67
+
68
+ Serving the real text lets the app FETCH the identity and cache it, so there
69
+ is one definition again. Truncating at 500 characters made that impossible,
70
+ which is why the app had a copy in the first place.
71
+ """
72
  try:
73
+ from modules.assistant_identity import (
74
+ JARVIS_PERSONALITY_PROMPT, FRIDAY_PERSONALITY_PROMPT,
75
+ get_identity_persona_prompt, get_mode, set_mode,
76
+ )
77
+ if not full:
78
+ return {
79
+ "jarvis": JARVIS_PERSONALITY_PROMPT[:500] + "...",
80
+ "friday": FRIDAY_PERSONALITY_PROMPT[:500] + "...",
81
+ }
82
+
83
+ # Composed exactly as every other consumer sees it — characterisation
84
+ # plus that persona's own tastes — so the phone gets the same identity
85
+ # the desktop brain and the backend agent get, not a subset.
86
+ original = get_mode()
87
+ out = {}
88
+ try:
89
+ for persona in ("jarvis", "friday"):
90
+ set_mode(persona)
91
+ out[persona] = get_identity_persona_prompt()
92
+ finally:
93
+ set_mode(original)
94
+
95
+ out["active"] = original
96
+ return out
97
  except Exception as e:
98
  raise HTTPException(status_code=500, detail=str(e))
backend/routes/token_task_routes.py ADDED
@@ -0,0 +1,94 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """backend/routes/token_task_routes.py — the token manager, reachable by the phone.
2
+
3
+ WHY THE PHONE MUST NOT HOLD THE KEYS
4
+ ------------------------------------
5
+ backend/services/token_manager.py routes each kind of work to its own API key
6
+ domain — OSINT_PROTOCOL_PC, RESEARCH_ENGINE, IMAGE_GENERATION_CLOUD and the
7
+ rest — so that one feature exhausting its quota cannot take the others down
8
+ with it. It also tracks exhausted keys, recovers them on a refresh monitor, and
9
+ checkpoints long tasks so they can resume.
10
+
11
+ The phone has none of that. MobileJarvisEngine.kt calls Gemini directly with a
12
+ single BuildConfig.GEMINI_API_KEY, which means: one key, no domain isolation,
13
+ no rotation, no recovery — and the key is compiled into the APK.
14
+
15
+ The fix is NOT to ship 37 keys to the handset. An APK carrying the master
16
+ credentials is personal-only and can never be distributed; it also puts every
17
+ key on a device that can be lost. Instead the phone asks the backend to do the
18
+ work, and the backend applies the full key strategy on its behalf. The handset
19
+ holds one scoped bearer token; the vault stays where it is.
20
+
21
+ This route is the door. It is deliberately thin: all the intelligence already
22
+ exists in token_manager.
23
+ """
24
+
25
+ from typing import Optional
26
+
27
+ from fastapi import APIRouter, HTTPException
28
+ from pydantic import BaseModel
29
+
30
+ router = APIRouter()
31
+
32
+
33
+ class TaskRequest(BaseModel):
34
+ prompt: str
35
+ # Selects the key DOMAIN, not the model. "osint" routes to the OSINT
36
+ # protocol keys, "research" to the research engine keys, and so on, which
37
+ # is the whole point of asking the backend rather than calling Gemini
38
+ # directly from the phone.
39
+ task_type: str = "general"
40
+ persona: str = "jarvis"
41
+
42
+
43
+ class TaskResponse(BaseModel):
44
+ status: str
45
+ result: str
46
+ task_type: str
47
+ persona: str
48
+
49
+
50
+ @router.post("/task", response_model=TaskResponse)
51
+ async def run_token_task(req: TaskRequest) -> TaskResponse:
52
+ """Run a prompt through the token manager, with domain-appropriate keys."""
53
+ prompt = (req.prompt or "").strip()
54
+ if not prompt:
55
+ raise HTTPException(status_code=400, detail="prompt is required")
56
+
57
+ try:
58
+ from backend.services.token_manager import run_task
59
+ result = await run_task(
60
+ prompt=prompt,
61
+ task_type=(req.task_type or "general").strip().lower(),
62
+ persona=(req.persona or "jarvis").strip().lower(),
63
+ )
64
+ return TaskResponse(
65
+ status="ok",
66
+ result=result or "",
67
+ task_type=req.task_type,
68
+ persona=req.persona,
69
+ )
70
+ except Exception as e:
71
+ raise HTTPException(status_code=500, detail=str(e))
72
+
73
+
74
+ @router.get("/key_status")
75
+ async def key_status() -> dict:
76
+ """What the pool looks like right now — counts only, never key material.
77
+
78
+ Useful from the phone precisely because the phone cannot see the vault: it
79
+ is the only way to tell "the feature is broken" from "that domain's quota
80
+ is spent and will recover".
81
+ """
82
+ try:
83
+ from backend.services import token_manager as tm
84
+ pool = getattr(tm, "GOOGLE_API_KEYS", {}) or {}
85
+ exhausted = getattr(tm, "EXHAUSTED_GOOGLE_KEYS", set()) or set()
86
+ return {
87
+ "status": "ok",
88
+ "domains_loaded": len(pool),
89
+ "domains": sorted(pool.keys()),
90
+ "exhausted_count": len(exhausted),
91
+ "healthy_count": max(0, len(pool) - len(exhausted)),
92
+ }
93
+ except Exception as e:
94
+ raise HTTPException(status_code=500, detail=str(e))
backend/services/token_manager.py CHANGED
@@ -1,669 +1,720 @@
1
- # backend/services/token_manager.py
2
- """
3
- OMEGA Token Limit Checkpoint & Resume System
4
-
5
- When any Gemini 3.5 Flash call hits a ResourceExhausted (429) token limit:
6
- 1. Saves the EXACT task state + all partial output generated so far to SQLite
7
- 2. Launches a background monitor that probes the API every 60s until tokens refresh
8
- 3. On refresh, resumes the task from the EXACT LAST WORD — not from the beginning
9
-
10
- All Gemini call sites (research, auto_upgrade, captcha, gate check) use
11
- gemini_call_with_checkpoint() instead of calling the API directly.
12
- """
13
-
14
- import asyncio
15
- import logging
16
- import os
17
- import sqlite3
18
- import time
19
- import uuid
20
- from typing import Optional
21
-
22
- import google.generativeai as genai
23
-
24
- from backend.services.usb_vault import KeyDomain, resolve_vault_key
25
-
26
- def get_runtime_location() -> str:
27
- return "cloud" if os.environ.get("SPACE_ID") else "pc"
28
-
29
- # Only the conversational path should be shaped in the assistant's voice.
30
- # Structured task types (blueprint JSON, code implementation, research digests,
31
- # captcha/osint payloads) must stay clean — a "You are JARVIS…" preamble there
32
- # corrupts JSON/code output.
33
- _PERSONA_TASK_TYPES = {"general", "gaming"}
34
-
35
- def _persona_system_prompt(persona: str, task_type: str = "general") -> str:
36
- """The identity prompt for the EXPLICITLY requested persona.
37
-
38
- The mobile app (and other callers) pass the persona they want per-turn, so
39
- we select on that string directly rather than the persisted global mode.
40
- Without this, conversational LLM replies carry no JARVIS/FRIDAY character —
41
- the identity file that is supposed to be the source of truth was being
42
- ignored on the primary /api/chat path. Returns "" for non-conversational
43
- task types so structured output is never polluted.
44
- """
45
- if task_type not in _PERSONA_TASK_TYPES:
46
- return ""
47
- try:
48
- from modules.assistant_identity import (
49
- JARVIS_PERSONALITY_PROMPT, FRIDAY_PERSONALITY_PROMPT,
50
- )
51
- p = (persona or "jarvis").strip().lower()
52
- return JARVIS_PERSONALITY_PROMPT if p == "jarvis" else FRIDAY_PERSONALITY_PROMPT
53
- except Exception:
54
- return ""
55
-
56
- # Load all 15 keys into the token manager pool
57
- GOOGLE_API_KEYS = {}
58
- try:
59
- for domain in KeyDomain:
60
- try:
61
- key_val = resolve_vault_key(domain)
62
- if key_val:
63
- GOOGLE_API_KEYS[domain.value] = key_val
64
- except Exception:
65
- continue
66
-
67
- if get_runtime_location() == "cloud":
68
- GEMINI_API_KEY = GOOGLE_API_KEYS.get(KeyDomain.CLOUD.value, "")
69
- else:
70
- GEMINI_API_KEY = GOOGLE_API_KEYS.get(KeyDomain.CHAT.value, "")
71
-
72
- if GEMINI_API_KEY:
73
- genai.configure(api_key=GEMINI_API_KEY)
74
- else:
75
- logging.warning("TokenManager: Default API Key is empty.")
76
-
77
- logging.info(f"TokenManager: Successfully loaded {len(GOOGLE_API_KEYS)} Google/Gemini API keys into the rotation pool.")
78
- except Exception as e:
79
- logging.warning(f"TokenManager: API Key initialization failed: {e}")
80
- GEMINI_API_KEY = ""
81
-
82
- EXHAUSTED_GOOGLE_KEYS = set()
83
- # Measured live against a real key, 2026-07-21:
84
- # gemini-3.5-flash HTTP 503 "currently experiencing high demand" <- was set here
85
- # gemini-2.0-flash HTTP 429 free-tier request cap
86
- # gemini-2.5-flash-lite HTTP 404 retired
87
- # gemini-3-flash-preview OK but 35.5s
88
- # gemini-3.1-flash-lite OK in 1.1s, correct answer <- chosen
89
- #
90
- # The 503 is Google-side capacity, NOT quota consumption: these keys were unused
91
- # for weeks. Because the primary model always failed, every request fell through
92
- # to the NVIDIA tier — which is why chat worked at all, and why research paid for
93
- # a doomed Gemini attempt before doing any real work.
94
- MODEL = "gemini-3.1-flash-lite"
95
-
96
- # Google-side model cascade, mirroring the NVIDIA tier cascade in nvidia_vault.
97
- #
98
- # A single hardcoded model is fragile in exactly two ways this system has already
99
- # been bitten by: the model gets retired (gemini-2.5-flash-lite -> 404) or it hits
100
- # Google-side capacity (gemini-3.5-flash -> 503 "high demand"). Neither means the
101
- # KEY is bad, and neither should cost a drop to the fallback tier — another
102
- # Gemini model is usually fine right now.
103
- #
104
- # Ordered by what was measured live 2026-07-21 against a real key:
105
- # gemini-3.1-flash-lite OK, 1.1s, correct answer <- best right now
106
- # gemini-3.5-flash 503 today, but transient — capacity comes and goes
107
- # gemini-3-flash-preview OK but 35.5s
108
- # gemini-2.0-flash 429 free-tier cap today
109
- # The list is tried in order on any transient failure, and only when every entry
110
- # fails does the request drop to the NVIDIA tier.
111
- GEMINI_MODEL_CASCADE = [
112
- MODEL,
113
- "gemini-3.5-flash",
114
- "gemini-3-flash-preview",
115
- "gemini-2.5-flash",
116
- "gemini-2.0-flash",
117
- ]
118
-
119
-
120
- def _is_transient_error(err_str: str) -> bool:
121
- """Transient upstream failure worth trying another model/tier."""
122
- return any(t in err_str for t in (
123
- "resourceexhausted", "429", "quota", "rate limit", "too many requests",
124
- "503", "500", "high demand", "overloaded", "unavailable",
125
- "internal error", "deadline", "timeout", "404", "not found",
126
- "no longer available",
127
- ))
128
-
129
- # ─────────────────────────────────────────────────────────────────────────────
130
- # DB Helpers
131
- # ─────────────────────────────────────────────────────────────────────────────
132
-
133
- from backend.services.usb_monitor import get_db_path
134
-
135
-
136
- def _init_checkpoint_table():
137
- """Ensure token_checkpoints table exists in memory.db."""
138
- try:
139
- with sqlite3.connect(get_db_path()) as conn:
140
- conn.execute("PRAGMA journal_mode=WAL")
141
- conn.execute("""
142
- CREATE TABLE IF NOT EXISTS token_checkpoints (
143
- id TEXT PRIMARY KEY,
144
- task_type TEXT NOT NULL,
145
- original_prompt TEXT NOT NULL,
146
- partial_result TEXT DEFAULT '',
147
- last_word TEXT DEFAULT '',
148
- status TEXT DEFAULT 'pending_resume',
149
- created_at INTEGER,
150
- resumed_at INTEGER,
151
- completed_at INTEGER,
152
- retry_count INTEGER DEFAULT 0,
153
- persona TEXT DEFAULT 'jarvis'
154
- )
155
- """)
156
- conn.commit()
157
- except Exception as e:
158
- logging.error(f"TokenManager: Failed to init checkpoint table: {e}")
159
-
160
-
161
- _init_checkpoint_table()
162
-
163
-
164
- def _save_checkpoint(task_id: str, task_type: str, prompt: str,
165
- partial: str, last_word: str, persona: str = "jarvis"):
166
- try:
167
- with sqlite3.connect(get_db_path()) as conn:
168
- conn.execute("PRAGMA journal_mode=WAL")
169
- conn.execute("""
170
- INSERT OR REPLACE INTO token_checkpoints
171
- (id, task_type, original_prompt, partial_result, last_word,
172
- status, created_at, persona)
173
- VALUES (?,?,?,?,?,'pending_resume',?,?)
174
- """, (task_id, task_type, prompt, partial, last_word,
175
- int(time.time()), persona))
176
- conn.commit()
177
- logging.warning(f"TokenManager: Checkpoint saved [{task_type}:{task_id}] last_word='{last_word}'")
178
- except Exception as e:
179
- logging.error(f"TokenManager: Failed to save checkpoint: {e}")
180
-
181
-
182
- def _mark_checkpoint_resumed(task_id: str):
183
- try:
184
- with sqlite3.connect(get_db_path()) as conn:
185
- conn.execute(
186
- "UPDATE token_checkpoints SET status='resumed', resumed_at=? WHERE id=?",
187
- (int(time.time()), task_id)
188
- )
189
- conn.commit()
190
- except Exception as e:
191
- logging.error(f"TokenManager: Failed to mark resumed: {e}")
192
-
193
-
194
- def _mark_checkpoint_complete(task_id: str):
195
- try:
196
- with sqlite3.connect(get_db_path()) as conn:
197
- conn.execute(
198
- "UPDATE token_checkpoints SET status='complete', completed_at=? WHERE id=?",
199
- (int(time.time()), task_id)
200
- )
201
- conn.commit()
202
- except Exception as e:
203
- logging.error(f"TokenManager: Failed to mark complete: {e}")
204
-
205
-
206
- def _get_pending_checkpoints() -> list:
207
- try:
208
- with sqlite3.connect(get_db_path()) as conn:
209
- conn.row_factory = sqlite3.Row
210
- rows = conn.execute(
211
- "SELECT * FROM token_checkpoints WHERE status='pending_resume' ORDER BY created_at ASC"
212
- ).fetchall()
213
- return [dict(r) for r in rows]
214
- except Exception as e:
215
- logging.error(f"TokenManager: Failed to fetch pending checkpoints: {e}")
216
- return []
217
-
218
-
219
- # ─────────────────────────────────────────────────────────────────────────────
220
- # Core Gemini Call — with auto checkpoint on 429
221
- # ─────────────────────────────────────────────────────────────────────────────
222
-
223
- def _extract_last_word(text: str) -> str:
224
- """Return the very last word of the partial output for precise resume anchoring."""
225
- if not text:
226
- return ""
227
- words = text.strip().split()
228
- return words[-1] if words else ""
229
-
230
-
231
- def _build_resume_prompt(original_prompt: str, partial_result: str, last_word: str) -> str:
232
- """
233
- Construct a continuation prompt so Gemini resumes from the EXACT last word.
234
- """
235
- if not partial_result:
236
- return original_prompt
237
- return (
238
- f"{original_prompt}\n\n"
239
- f"---OMEGA RESUME INSTRUCTION---\n"
240
- f"You were previously generating a response and hit a token limit mid-way.\n"
241
- f"The partial output generated so far ended with the word: '{last_word}'\n"
242
- f"Here is the partial output so far:\n\n{partial_result}\n\n"
243
- f"Continue EXACTLY from where you left off. "
244
- f"Do NOT repeat any of the partial output above. "
245
- f"Start your response from the word that comes AFTER '{last_word}'."
246
- )
247
-
248
-
249
- def _get_key_for_task(task_type: str) -> str:
250
- from backend.services.usb_vault import KeyDomain, resolve_vault_key
251
- location = get_runtime_location()
252
-
253
- if task_type == "research":
254
- return resolve_vault_key(KeyDomain.RESEARCH_ENGINE)
255
- elif task_type == "captcha":
256
- return resolve_vault_key(KeyDomain.CAPTCHA_SOLVER_CLOUD if location == "cloud" else KeyDomain.CAPTCHA_SOLVER_PC)
257
- elif task_type == "implement":
258
- return resolve_vault_key(KeyDomain.AUTO_UPGRADE_CLOUD if location == "cloud" else KeyDomain.AUTO_UPGRADE_PC)
259
- elif task_type == "osint":
260
- return resolve_vault_key(KeyDomain.OSINT_PROTOCOL_CLOUD if location == "cloud" else KeyDomain.OSINT_PROTOCOL_PC)
261
- elif task_type == "image":
262
- return resolve_vault_key(KeyDomain.IMAGE_GENERATION_CLOUD if location == "cloud" else KeyDomain.IMAGE_GENERATION_PC)
263
- elif task_type == "gaming":
264
- return resolve_vault_key(KeyDomain.GAMING_COACH)
265
- elif task_type == "supervisor":
266
- return resolve_vault_key(KeyDomain.SUPERVISOR_HEAL_CLOUD if location == "cloud" else KeyDomain.SUPERVISOR_HEAL_PC)
267
- elif task_type == "cloud_sync":
268
- return resolve_vault_key(KeyDomain.CLOUD_SYNC_DOMAIN)
269
- else:
270
- return resolve_vault_key(KeyDomain.CLOUD if location == "cloud" else KeyDomain.CHAT)
271
-
272
- def _broadcast_live_key_status(ecosystem: str, model: str, status: str):
273
- """Safely fire WS broadcast from synchronous threads"""
274
- payload = {
275
- "event": "live_key_status",
276
- "payload": {
277
- "active_ecosystem": ecosystem,
278
- "active_model": model,
279
- "status": status
280
- }
281
- }
282
- try:
283
- from backend.ws.agent_ws import ws_manager
284
- loop = asyncio.get_running_loop()
285
- loop.create_task(ws_manager.broadcast(payload))
286
- except RuntimeError:
287
- pass # No loop running in this thread
288
-
289
- def _nvidia_fallback_call_sync(prompt: str, task_id: str, task_type: str, partial_so_far: str, persona: str) -> str:
290
- """Fallback to NVIDIA Vault Models."""
291
- _broadcast_live_key_status("NVIDIA", "glm-5.1", "fallback_mode")
292
-
293
- from backend.services.memory_service import GlobalOmniMemory
294
- omni_context = GlobalOmniMemory.get_global_context_stream()
295
-
296
- # Construct exact prompt with resume logic and hive mind context
297
- actual_prompt = _build_resume_prompt(prompt, partial_so_far, _extract_last_word(partial_so_far))
298
- if not partial_so_far:
299
- # NVIDIA models take no separate system channel here, so the persona
300
- # prompt is prepended (same place the hive-mind context goes).
301
- _sys = _persona_system_prompt(persona, task_type)
302
- actual_prompt = (_sys + "\n\n" if _sys else "") + omni_context + actual_prompt
303
-
304
- # Delegate to the NVIDIA Vault API directly
305
- from backend.services.nvidia_vault import call_nvidia_model, NvidiaModel
306
- # GLM is the intended head of the heavy-compute fallback tier. call_nvidia_model
307
- # already cascades from here through the rest of that tier (MiniMax, DeepSeek,
308
- # Nemotron-Ultra) with up to 3 key-failovers per model, so a single slow or
309
- # unavailable model does not strand the request.
310
- #
311
- # Only the model ID changed: "z-ai/glm-5.1" was retired upstream and returned
312
- # HTTP 410 on all 15 keys, which wasted the first attempt of every heavy-tier
313
- # call. It is now glm-5.2.
314
- fallback_response = call_nvidia_model(actual_prompt, NvidiaModel.GLM_5_2)
315
-
316
- # Record what NVIDIA just did to the Hive Mind
317
- GlobalOmniMemory.record_action("NVIDIA", "glm-5.1", task_type, fallback_response[:100].replace('\n', ' ') + "...")
318
-
319
- combined = partial_so_far + fallback_response
320
- _mark_checkpoint_complete(task_id)
321
- return combined
322
-
323
- def _gemini_call_sync(prompt: str, task_id: str, task_type: str,
324
- partial_so_far: str = "", persona: str = "jarvis") -> str:
325
- key = _get_key_for_task(task_type)
326
-
327
- # 1. If we know it's exhausted, fallback immediately
328
- if key in EXHAUSTED_GOOGLE_KEYS:
329
- logging.info(f"TokenManager: Google Key exhausted. Routing {task_type} instantly to NVIDIA Fallback.")
330
- return _nvidia_fallback_call_sync(prompt, task_id, task_type, partial_so_far, persona)
331
-
332
- genai.configure(api_key=key)
333
- _broadcast_live_key_status("GOOGLE", MODEL, "primary_active")
334
-
335
- from backend.services.memory_service import GlobalOmniMemory
336
- omni_context = GlobalOmniMemory.get_global_context_stream()
337
-
338
- # Shape the reply in the active assistant's voice (JARVIS/FRIDAY). Skip on a
339
- # resume continuation so we don't restate the persona mid-sentence.
340
- _sys = _persona_system_prompt(persona, task_type) if not partial_so_far else None
341
- actual_prompt = _build_resume_prompt(prompt, partial_so_far, _extract_last_word(partial_so_far))
342
- if not partial_so_far:
343
- actual_prompt = omni_context + actual_prompt
344
-
345
- # Walk the Gemini cascade before considering the NVIDIA tier. A model that is
346
- # retired or momentarily at capacity must not cost the whole Google side —
347
- # the next model in the list is usually healthy.
348
- last_exc = None
349
- for attempt_model in GEMINI_MODEL_CASCADE:
350
- try:
351
- model = (genai.GenerativeModel(attempt_model, system_instruction=_sys)
352
- if _sys else genai.GenerativeModel(attempt_model))
353
- response = model.generate_content(actual_prompt)
354
- new_text = response.text or ""
355
-
356
- if attempt_model != MODEL:
357
- logging.warning("TokenManager: primary model unavailable; served by %s", attempt_model)
358
- _broadcast_live_key_status("GOOGLE", attempt_model, "primary_active")
359
-
360
- # Record what Google just did to the Hive Mind
361
- GlobalOmniMemory.record_action("GOOGLE", attempt_model, task_type,
362
- new_text[:100].replace('\n', ' ') + "...")
363
-
364
- combined = partial_so_far + new_text
365
- _mark_checkpoint_complete(task_id)
366
- return combined
367
- except Exception as exc:
368
- last_exc = exc
369
- if _is_transient_error(str(exc).lower()):
370
- logging.info("TokenManager: %s unavailable (%s); trying next Gemini model.",
371
- attempt_model, str(exc)[:70])
372
- continue
373
- raise
374
-
375
- # Every Gemini model failed — fall through to the original handling, which
376
- # decides between key-exhaustion bookkeeping and the NVIDIA tier.
377
- try:
378
- raise last_exc
379
- except Exception as e:
380
- err_str = str(e).lower()
381
- # Any TRANSIENT upstream failure should fall back to NVIDIA, not just a
382
- # quota error. This previously matched 429/quota only, so a Google-side
383
- # capacity 503 ("This model is currently experiencing high demand" — which
384
- # gemini-3.5-flash returns constantly) fell through to `raise` and killed
385
- # the request outright, never reaching the NVIDIA tier that exists for
386
- # exactly this situation.
387
- is_rate_limit = (
388
- "resourceexhausted" in err_str
389
- or "429" in err_str
390
- or "quota" in err_str
391
- or "rate limit" in err_str
392
- or "too many requests" in err_str
393
- # Google-side capacity / availability, not our usage:
394
- or "503" in err_str
395
- or "500" in err_str
396
- or "high demand" in err_str
397
- or "overloaded" in err_str
398
- or "unavailable" in err_str
399
- or "internal error" in err_str
400
- or "deadline" in err_str
401
- or "timeout" in err_str
402
- )
403
- # A capacity/availability failure is the MODEL's problem, not the key's.
404
- # Blacklisting the key for a 503 would permanently route a perfectly
405
- # healthy credential to the fallback tier — so only real quota errors
406
- # mark a key exhausted. Both still fall back to NVIDIA.
407
- is_key_exhausted = (
408
- "resourceexhausted" in err_str
409
- or "429" in err_str
410
- or "quota" in err_str
411
- or "rate limit" in err_str
412
- or "too many requests" in err_str
413
- )
414
- if is_rate_limit:
415
- logging.warning(
416
- "TokenManager: transient upstream failure for [%s:%s] (%s). Triggering NVIDIA Fallback.",
417
- task_type, task_id, "quota" if is_key_exhausted else "capacity/503",
418
- )
419
- if not is_key_exhausted:
420
- # Model-side outage: keep the key healthy, just serve this request
421
- # from the fallback tier.
422
- return _nvidia_fallback_call_sync(prompt, task_id, task_type, partial_so_far, persona)
423
- # 2. Add to exhausted pool
424
- if key not in EXHAUSTED_GOOGLE_KEYS:
425
- EXHAUSTED_GOOGLE_KEYS.add(key)
426
-
427
- # We can't spawn an async task cleanly from a sync thread without a loop,
428
- # so we will raise a special exception that the async wrapper catches to spawn the pinger.
429
- raise _Google429Trigger(key=key, prompt=prompt, task_id=task_id, task_type=task_type, partial_so_far=partial_so_far, persona=persona)
430
-
431
- return _nvidia_fallback_call_sync(prompt, task_id, task_type, partial_so_far, persona)
432
- raise
433
-
434
- class _Google429Trigger(Exception):
435
- def __init__(self, key, prompt, task_id, task_type, partial_so_far, persona):
436
- self.key = key
437
- self.prompt = prompt
438
- self.task_id = task_id
439
- self.task_type = task_type
440
- self.partial_so_far = partial_so_far
441
- self.persona = persona
442
-
443
- TokenLimitHit = _Google429Trigger
444
-
445
- async def gemini_call_with_checkpoint(
446
- prompt: str,
447
- task_type: str = "general",
448
- persona: str = "jarvis",
449
- task_id: Optional[str] = None,
450
- partial_so_far: str = ""
451
- ) -> str:
452
- """
453
- Universal Gemini 3.5 Flash call with automatic checkpoint & resume.
454
- """
455
- if task_id is None:
456
- task_id = str(uuid.uuid4())
457
-
458
- try:
459
- return await asyncio.to_thread(
460
- _gemini_call_sync, prompt, task_id, task_type, partial_so_far, persona
461
- )
462
- except _Google429Trigger as e:
463
- # 1. Start the token refresh monitor loop
464
- asyncio.create_task(start_token_refresh_monitor())
465
- # 2. Immediately complete THIS task using NVIDIA fallback
466
- return await asyncio.to_thread(
467
- _nvidia_fallback_call_sync, e.prompt, e.task_id, e.task_type, e.partial_so_far, e.persona
468
- )
469
-
470
- # For backward compatibility where 'run_task' is imported from token_manager
471
- async def run_task(prompt: str, task_type: str = "general", persona: str = "jarvis") -> str:
472
- return await gemini_call_with_checkpoint(prompt, task_type, persona)
473
-
474
-
475
- # ─────────────────────────────────────────────────────────────────────────────
476
- # Background Cloud Monitor — polls until tokens refresh then resumes
477
- # ─────────────────────────────────────────────────────────────────────────────
478
-
479
- _monitor_running = False
480
-
481
- async def start_token_refresh_monitor():
482
- """
483
- Background loop that:
484
- 1. Checks for pending_resume checkpoints every 60s
485
- 2. Probes Gemini with a minimal token-cost ping
486
- 3. When the probe succeeds, resumes ALL pending tasks from their exact last word
487
- 4. Broadcasts omega:task_resumed WS event for each resumed task
488
- """
489
- global _monitor_running
490
- if _monitor_running:
491
- return
492
- _monitor_running = True
493
- logging.info("TokenManager: Token refresh monitor STARTED.")
494
-
495
- while _monitor_running:
496
- try:
497
- pending = _get_pending_checkpoints()
498
-
499
- # We must probe if there are pending checkpoints OR if we have exhausted keys that need refreshing
500
- if pending or EXHAUSTED_GOOGLE_KEYS:
501
- if pending:
502
- logging.info(f"TokenManager: {len(pending)} checkpoint(s) pending resume. Probing API...")
503
- else:
504
- logging.info("TokenManager: No checkpoints, but EXHAUSTED_GOOGLE_KEYS has entries. Probing API...")
505
-
506
- # Lightweight probe — minimal cost
507
- # Check each exhausted key independently
508
- keys_to_probe = list(EXHAUSTED_GOOGLE_KEYS) if EXHAUSTED_GOOGLE_KEYS else [_get_key_for_task("general")]
509
- recovered_keys = []
510
-
511
- for key in keys_to_probe:
512
- try:
513
- genai.configure(api_key=key)
514
- model = genai.GenerativeModel(MODEL)
515
- model.generate_content("ping")
516
- recovered_keys.append(key)
517
- except Exception as probe_err:
518
- err_str = str(probe_err).lower()
519
- if "resourceexhausted" in err_str or "429" in err_str or "quota" in err_str:
520
- pass # Still exhausted
521
- else:
522
- logging.error(f"TokenManager: Probe error on key: {probe_err}")
523
-
524
- if not recovered_keys and keys_to_probe:
525
- logging.warning("TokenManager: API still exhausted. Will retry in 60s.")
526
-
527
- if recovered_keys:
528
- logging.info(f"TokenManager: API Probe OK for {len(recovered_keys)} keys. Clearing them from EXHAUSTED_GOOGLE_KEYS.")
529
- for k in recovered_keys:
530
- if k in EXHAUSTED_GOOGLE_KEYS:
531
- EXHAUSTED_GOOGLE_KEYS.remove(k)
532
-
533
- if pending:
534
- # Resume all pending checkpoints now that some keys are refreshed
535
- for checkpoint in pending:
536
- asyncio.create_task(_resume_checkpoint(checkpoint))
537
-
538
- except Exception as e:
539
- logging.error(f"TokenManager: Monitor loop error: {e}")
540
-
541
- await asyncio.sleep(60)
542
-
543
- logging.info("TokenManager: Token refresh monitor STOPPED.")
544
-
545
-
546
- async def _resume_checkpoint(checkpoint: dict):
547
- """Resume a single checkpoint from its exact last word."""
548
- task_id = checkpoint["id"]
549
- task_type = checkpoint["task_type"]
550
- original_prompt = checkpoint["original_prompt"]
551
- partial = checkpoint["partial_result"] or ""
552
- last_word = checkpoint["last_word"] or ""
553
- persona = checkpoint.get("persona", "jarvis")
554
-
555
- logging.info(f"TokenManager: Resuming [{task_type}:{task_id}] from last_word='{last_word}'")
556
- _mark_checkpoint_resumed(task_id)
557
-
558
- try:
559
- # Broadcast WS notification
560
- from backend.ws.agent_ws import ws_manager
561
- await ws_manager.broadcast({
562
- "event": "omega:task_resuming",
563
- "payload": {
564
- "task_id": task_id,
565
- "task_type": task_type,
566
- "last_word": last_word,
567
- "message": (
568
- f"Token limit has refreshed, sir. Resuming {task_type} task from exactly '{last_word}'..."
569
- if persona == "jarvis" else
570
- f"We're back boss! Tokens refreshed resuming {task_type} from exactly '{last_word}'!"
571
- )
572
- }
573
- })
574
-
575
- # Re-run the Gemini call from exact checkpoint
576
- result = await gemini_call_with_checkpoint(
577
- prompt=original_prompt,
578
- task_type=task_type,
579
- persona=persona,
580
- task_id=task_id,
581
- partial_so_far=partial
582
- )
583
-
584
- _mark_checkpoint_complete(task_id)
585
-
586
- # Route result back to its originating system
587
- await _dispatch_resumed_result(task_type, task_id, result, persona, checkpoint)
588
-
589
- await ws_manager.broadcast({
590
- "event": "omega:task_resumed",
591
- "payload": {
592
- "task_id": task_id,
593
- "task_type": task_type,
594
- "message": (
595
- f"{task_type.title()} task completed after token refresh, sir."
596
- if persona == "jarvis" else
597
- f"Done boss! {task_type.title()} task finished after the token refresh!"
598
- )
599
- }
600
- })
601
-
602
- except TokenLimitHit:
603
- # Still throttled will retry next monitor cycle
604
- logging.warning(f"TokenManager: Still throttled on resume for [{task_id}]. Reverting to pending_resume.")
605
- try:
606
- with sqlite3.connect(get_db_path()) as conn:
607
- conn.execute(
608
- "UPDATE token_checkpoints SET status='pending_resume', retry_count=retry_count+1 WHERE id=?",
609
- (task_id,)
610
- )
611
- conn.commit()
612
- except Exception as e:
613
- # No local `import logging` — it would shadow the module import for
614
- # the whole of _resume_checkpoint() and break the logging calls above.
615
- logging.getLogger(__name__).error(f"Swallowed exception: {e}")
616
- except Exception as e:
617
- logging.error(f"TokenManager: Resume error for [{task_id}]: {e}")
618
-
619
-
620
- async def _dispatch_resumed_result(task_type: str, task_id: str,
621
- result: str, persona: str, checkpoint: dict):
622
- """Route the completed result back to the correct subsystem."""
623
- try:
624
- if task_type == "research":
625
- # Re-parse the JSON research note and save it
626
- import json
627
- from backend.omega.research_engine import ResearchNote, ResearchCategory, _save_research_note
628
- raw = result.strip().strip("```json").strip("```").strip()
629
- data = json.loads(raw)
630
- raw_category = data.get("category", "Research Notes")
631
- valid_categories = {c.value: c for c in ResearchCategory}
632
- category = valid_categories.get(raw_category, ResearchCategory.RESEARCH_NOTES)
633
- note = ResearchNote(
634
- title=data.get("title", "Resumed Research"),
635
- summary=data.get("summary", result[:500]),
636
- importance=data.get("importance", "MEDIUM"),
637
- category=category,
638
- source=data.get("source", "Gemini 3.5 Flash (resumed)"),
639
- recommended_action=data.get("recommended_action", "")
640
- )
641
- _save_research_note(note)
642
- logging.info(f"TokenManager: Research note saved after resume: {note.title}")
643
-
644
- elif task_type == "implement":
645
- # Re-parse code blocks and hot-reload them
646
- from backend.omega.auto_upgrade import parse_code_blocks, write_and_hot_reload, log_upgrade_to_db
647
- code_blocks = parse_code_blocks(result)
648
- files_modified = []
649
- for block in code_blocks:
650
- await write_and_hot_reload(block.filepath, block.code)
651
- files_modified.append(block.filepath)
652
- log_upgrade_to_db(
653
- checkpoint.get("original_prompt", "Resumed task")[:60],
654
- files_modified, persona, "success_after_resume"
655
- )
656
- logging.info(f"TokenManager: Implementation resumed and hot-reloaded: {files_modified}")
657
-
658
- else:
659
- # For gate/captcha/monitor tasks, just log the full result
660
- logging.info(f"TokenManager: [{task_type}] resumed result stored (length={len(result)})")
661
-
662
- except Exception as e:
663
- logging.error(f"TokenManager: dispatch_resumed_result error for [{task_type}]: {e}")
664
-
665
-
666
- def stop_token_refresh_monitor():
667
- global _monitor_running
668
- _monitor_running = False
669
- logging.info("TokenManager: Token refresh monitor signalled to stop.")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # backend/services/token_manager.py
2
+ """
3
+ OMEGA Token Limit Checkpoint & Resume System
4
+
5
+ When any Gemini 3.5 Flash call hits a ResourceExhausted (429) token limit:
6
+ 1. Saves the EXACT task state + all partial output generated so far to SQLite
7
+ 2. Launches a background monitor that probes the API every 60s until tokens refresh
8
+ 3. On refresh, resumes the task from the EXACT LAST WORD — not from the beginning
9
+
10
+ All Gemini call sites (research, auto_upgrade, captcha, gate check) use
11
+ gemini_call_with_checkpoint() instead of calling the API directly.
12
+ """
13
+
14
+ import asyncio
15
+ import logging
16
+ import os
17
+ import sqlite3
18
+ import time
19
+ import uuid
20
+ from typing import Optional
21
+
22
+ import google.generativeai as genai
23
+
24
+ from backend.services.usb_vault import KeyDomain, resolve_vault_key
25
+
26
+ def get_runtime_location() -> str:
27
+ return "cloud" if os.environ.get("SPACE_ID") else "pc"
28
+
29
+ # Only the conversational path should be shaped in the assistant's voice.
30
+ # Structured task types (blueprint JSON, code implementation, research digests,
31
+ # captcha/osint payloads) must stay clean — a "You are JARVIS…" preamble there
32
+ # corrupts JSON/code output.
33
+ _PERSONA_TASK_TYPES = {"general", "gaming"}
34
+
35
+ def _persona_system_prompt(persona: str, task_type: str = "general",
36
+ live_state: Optional[dict] = None,
37
+ can_search: bool = False,
38
+ can_act: bool = False,
39
+ can_read: bool = False) -> str:
40
+ """The identity prompt for the EXPLICITLY requested persona.
41
+
42
+ The mobile app (and other callers) pass the persona they want per-turn, so
43
+ we select on that string directly rather than the persisted global mode.
44
+ Without this, conversational LLM replies carry no JARVIS/FRIDAY character —
45
+ the identity file that is supposed to be the source of truth was being
46
+ ignored on the primary /api/chat path. Returns "" for non-conversational
47
+ task types so structured output is never polluted.
48
+
49
+ It now composes the FULL identity — characterisation, tastes, behavioural
50
+ judgement and measured live state — rather than the characterisation alone.
51
+ Serving only the characterisation is what produced a JARVIS who opened every
52
+ reply with "at your service", wrote markdown asterisks into speech, and
53
+ announced the PC was connected while it was powered off: he had the
54
+ personality of the character and none of his situational sense, and no facts
55
+ at all. compose_identity() is shared with the desktop brain so the two
56
+ cannot drift apart again.
57
+ """
58
+ if task_type not in _PERSONA_TASK_TYPES:
59
+ return ""
60
+ try:
61
+ from modules.assistant_identity import compose_identity
62
+ return compose_identity(persona=persona, live_state=live_state,
63
+ can_search=can_search, can_act=can_act, can_read=can_read)
64
+ except Exception:
65
+ # Never lose the identity entirely because state-gathering broke.
66
+ try:
67
+ from modules.assistant_identity import (
68
+ JARVIS_PERSONALITY_PROMPT, FRIDAY_PERSONALITY_PROMPT,
69
+ )
70
+ p = (persona or "jarvis").strip().lower()
71
+ return JARVIS_PERSONALITY_PROMPT if p == "jarvis" else FRIDAY_PERSONALITY_PROMPT
72
+ except Exception:
73
+ return ""
74
+
75
+ # Load all 15 keys into the token manager pool
76
+ GOOGLE_API_KEYS = {}
77
+ try:
78
+ for domain in KeyDomain:
79
+ try:
80
+ key_val = resolve_vault_key(domain)
81
+ if key_val:
82
+ GOOGLE_API_KEYS[domain.value] = key_val
83
+ except Exception:
84
+ continue
85
+
86
+ if get_runtime_location() == "cloud":
87
+ GEMINI_API_KEY = GOOGLE_API_KEYS.get(KeyDomain.CLOUD.value, "")
88
+ else:
89
+ GEMINI_API_KEY = GOOGLE_API_KEYS.get(KeyDomain.CHAT.value, "")
90
+
91
+ if GEMINI_API_KEY:
92
+ genai.configure(api_key=GEMINI_API_KEY)
93
+ else:
94
+ logging.warning("TokenManager: Default API Key is empty.")
95
+
96
+ logging.info(f"TokenManager: Successfully loaded {len(GOOGLE_API_KEYS)} Google/Gemini API keys into the rotation pool.")
97
+ except Exception as e:
98
+ logging.warning(f"TokenManager: API Key initialization failed: {e}")
99
+ GEMINI_API_KEY = ""
100
+
101
+ EXHAUSTED_GOOGLE_KEYS = set()
102
+ # Measured live against a real key, 2026-07-21:
103
+ # gemini-3.5-flash HTTP 503 "currently experiencing high demand" <- was set here
104
+ # gemini-2.0-flash HTTP 429 free-tier request cap
105
+ # gemini-2.5-flash-lite HTTP 404 retired
106
+ # gemini-3-flash-preview OK but 35.5s
107
+ # gemini-3.1-flash-lite OK in 1.1s, correct answer <- chosen
108
+ #
109
+ # The 503 is Google-side capacity, NOT quota consumption: these keys were unused
110
+ # for weeks. Because the primary model always failed, every request fell through
111
+ # to the NVIDIA tier — which is why chat worked at all, and why research paid for
112
+ # a doomed Gemini attempt before doing any real work.
113
+ MODEL = "gemini-3.1-flash-lite"
114
+
115
+ # Google-side model cascade, mirroring the NVIDIA tier cascade in nvidia_vault.
116
+ #
117
+ # A single hardcoded model is fragile in exactly two ways this system has already
118
+ # been bitten by: the model gets retired (gemini-2.5-flash-lite -> 404) or it hits
119
+ # Google-side capacity (gemini-3.5-flash -> 503 "high demand"). Neither means the
120
+ # KEY is bad, and neither should cost a drop to the fallback tier — another
121
+ # Gemini model is usually fine right now.
122
+ #
123
+ # Ordered by what was measured live 2026-07-21 against a real key:
124
+ # gemini-3.1-flash-lite OK, 1.1s, correct answer <- best right now
125
+ # gemini-3.5-flash 503 today, but transient capacity comes and goes
126
+ # gemini-3-flash-preview OK but 35.5s
127
+ # gemini-2.0-flash 429 free-tier cap today
128
+ # The list is tried in order on any transient failure, and only when every entry
129
+ # fails does the request drop to the NVIDIA tier.
130
+ #
131
+ # Re-verified 2026-08-13 by listing models AND calling each one. Two corrections:
132
+ # gemini-2.0-flash GONE — not in list_models at all, dropped
133
+ # gemini-2.5-flash-lite listed as available, answers 404 on generateContent
134
+ # The second is the trap: list_models is not proof the account may CALL a model,
135
+ # so a dead entry can sit here looking healthy while costing a round trip and a
136
+ # retry on every failover. Verify by calling, never by listing.
137
+ GEMINI_MODEL_CASCADE = [
138
+ MODEL, # gemini-3.1-flash-lite — fastest verified
139
+ "gemini-3.5-flash-lite",
140
+ "gemini-3.5-flash",
141
+ "gemini-2.5-flash",
142
+ "gemini-3-flash-preview", # works, but ~35s
143
+ ]
144
+
145
+
146
+ def _is_transient_error(err_str: str) -> bool:
147
+ """Transient upstream failure — worth trying another model/tier."""
148
+ return any(t in err_str for t in (
149
+ "resourceexhausted", "429", "quota", "rate limit", "too many requests",
150
+ "503", "500", "high demand", "overloaded", "unavailable",
151
+ "internal error", "deadline", "timeout", "404", "not found",
152
+ "no longer available",
153
+ ))
154
+
155
+ # ─────────────────────────────────────────────────────────────────────────────
156
+ # DB Helpers
157
+ # ─────────────────────────────────────────────────────────────────────────────
158
+
159
+ from backend.services.usb_monitor import get_db_path
160
+
161
+
162
+ def _init_checkpoint_table():
163
+ """Ensure token_checkpoints table exists in memory.db."""
164
+ try:
165
+ with sqlite3.connect(get_db_path()) as conn:
166
+ conn.execute("PRAGMA journal_mode=WAL")
167
+ conn.execute("""
168
+ CREATE TABLE IF NOT EXISTS token_checkpoints (
169
+ id TEXT PRIMARY KEY,
170
+ task_type TEXT NOT NULL,
171
+ original_prompt TEXT NOT NULL,
172
+ partial_result TEXT DEFAULT '',
173
+ last_word TEXT DEFAULT '',
174
+ status TEXT DEFAULT 'pending_resume',
175
+ created_at INTEGER,
176
+ resumed_at INTEGER,
177
+ completed_at INTEGER,
178
+ retry_count INTEGER DEFAULT 0,
179
+ persona TEXT DEFAULT 'jarvis'
180
+ )
181
+ """)
182
+ conn.commit()
183
+ except Exception as e:
184
+ logging.error(f"TokenManager: Failed to init checkpoint table: {e}")
185
+
186
+
187
+ _init_checkpoint_table()
188
+
189
+
190
+ def _save_checkpoint(task_id: str, task_type: str, prompt: str,
191
+ partial: str, last_word: str, persona: str = "jarvis"):
192
+ try:
193
+ with sqlite3.connect(get_db_path()) as conn:
194
+ conn.execute("PRAGMA journal_mode=WAL")
195
+ conn.execute("""
196
+ INSERT OR REPLACE INTO token_checkpoints
197
+ (id, task_type, original_prompt, partial_result, last_word,
198
+ status, created_at, persona)
199
+ VALUES (?,?,?,?,?,'pending_resume',?,?)
200
+ """, (task_id, task_type, prompt, partial, last_word,
201
+ int(time.time()), persona))
202
+ conn.commit()
203
+ logging.warning(f"TokenManager: Checkpoint saved [{task_type}:{task_id}] last_word='{last_word}'")
204
+ except Exception as e:
205
+ logging.error(f"TokenManager: Failed to save checkpoint: {e}")
206
+
207
+
208
+ def _mark_checkpoint_resumed(task_id: str):
209
+ try:
210
+ with sqlite3.connect(get_db_path()) as conn:
211
+ conn.execute(
212
+ "UPDATE token_checkpoints SET status='resumed', resumed_at=? WHERE id=?",
213
+ (int(time.time()), task_id)
214
+ )
215
+ conn.commit()
216
+ except Exception as e:
217
+ logging.error(f"TokenManager: Failed to mark resumed: {e}")
218
+
219
+
220
+ def _mark_checkpoint_complete(task_id: str):
221
+ try:
222
+ with sqlite3.connect(get_db_path()) as conn:
223
+ conn.execute(
224
+ "UPDATE token_checkpoints SET status='complete', completed_at=? WHERE id=?",
225
+ (int(time.time()), task_id)
226
+ )
227
+ conn.commit()
228
+ except Exception as e:
229
+ logging.error(f"TokenManager: Failed to mark complete: {e}")
230
+
231
+
232
+ def _get_pending_checkpoints() -> list:
233
+ try:
234
+ with sqlite3.connect(get_db_path()) as conn:
235
+ conn.row_factory = sqlite3.Row
236
+ rows = conn.execute(
237
+ "SELECT * FROM token_checkpoints WHERE status='pending_resume' ORDER BY created_at ASC"
238
+ ).fetchall()
239
+ return [dict(r) for r in rows]
240
+ except Exception as e:
241
+ logging.error(f"TokenManager: Failed to fetch pending checkpoints: {e}")
242
+ return []
243
+
244
+
245
+ # ─────────────────────────────────────────────────────────────────────────────
246
+ # Core Gemini Call — with auto checkpoint on 429
247
+ # ─────────────────────────────────────────────────────────────────────────────
248
+
249
+ def _extract_last_word(text: str) -> str:
250
+ """Return the very last word of the partial output for precise resume anchoring."""
251
+ if not text:
252
+ return ""
253
+ words = text.strip().split()
254
+ return words[-1] if words else ""
255
+
256
+
257
+ def _build_resume_prompt(original_prompt: str, partial_result: str, last_word: str) -> str:
258
+ """
259
+ Construct a continuation prompt so Gemini resumes from the EXACT last word.
260
+ """
261
+ if not partial_result:
262
+ return original_prompt
263
+ return (
264
+ f"{original_prompt}\n\n"
265
+ f"---OMEGA RESUME INSTRUCTION---\n"
266
+ f"You were previously generating a response and hit a token limit mid-way.\n"
267
+ f"The partial output generated so far ended with the word: '{last_word}'\n"
268
+ f"Here is the partial output so far:\n\n{partial_result}\n\n"
269
+ f"Continue EXACTLY from where you left off. "
270
+ f"Do NOT repeat any of the partial output above. "
271
+ f"Start your response from the word that comes AFTER '{last_word}'."
272
+ )
273
+
274
+
275
+ def _get_key_for_task(task_type: str) -> str:
276
+ from backend.services.usb_vault import KeyDomain, resolve_vault_key
277
+ location = get_runtime_location()
278
+
279
+ if task_type == "research":
280
+ return resolve_vault_key(KeyDomain.RESEARCH_ENGINE)
281
+ elif task_type == "captcha":
282
+ return resolve_vault_key(KeyDomain.CAPTCHA_SOLVER_CLOUD if location == "cloud" else KeyDomain.CAPTCHA_SOLVER_PC)
283
+ elif task_type == "implement":
284
+ return resolve_vault_key(KeyDomain.AUTO_UPGRADE_CLOUD if location == "cloud" else KeyDomain.AUTO_UPGRADE_PC)
285
+ elif task_type == "osint":
286
+ return resolve_vault_key(KeyDomain.OSINT_PROTOCOL_CLOUD if location == "cloud" else KeyDomain.OSINT_PROTOCOL_PC)
287
+ elif task_type == "image":
288
+ return resolve_vault_key(KeyDomain.IMAGE_GENERATION_CLOUD if location == "cloud" else KeyDomain.IMAGE_GENERATION_PC)
289
+ elif task_type == "gaming":
290
+ return resolve_vault_key(KeyDomain.GAMING_COACH)
291
+ elif task_type == "supervisor":
292
+ return resolve_vault_key(KeyDomain.SUPERVISOR_HEAL_CLOUD if location == "cloud" else KeyDomain.SUPERVISOR_HEAL_PC)
293
+ elif task_type == "cloud_sync":
294
+ return resolve_vault_key(KeyDomain.CLOUD_SYNC_DOMAIN)
295
+ else:
296
+ return resolve_vault_key(KeyDomain.CLOUD if location == "cloud" else KeyDomain.CHAT)
297
+
298
+ def _broadcast_live_key_status(ecosystem: str, model: str, status: str):
299
+ """Safely fire WS broadcast from synchronous threads"""
300
+ payload = {
301
+ "event": "live_key_status",
302
+ "payload": {
303
+ "active_ecosystem": ecosystem,
304
+ "active_model": model,
305
+ "status": status
306
+ }
307
+ }
308
+ try:
309
+ from backend.ws.agent_ws import ws_manager
310
+ loop = asyncio.get_running_loop()
311
+ loop.create_task(ws_manager.broadcast(payload))
312
+ except RuntimeError:
313
+ pass # No loop running in this thread
314
+
315
+ def _nvidia_fallback_call_sync(prompt: str, task_id: str, task_type: str, partial_so_far: str,
316
+ persona: str, live_state: Optional[dict] = None,
317
+ can_search: bool = False, can_act: bool = False,
318
+ can_read: bool = False) -> str:
319
+ """Fallback to NVIDIA Vault Models."""
320
+ _broadcast_live_key_status("NVIDIA", "glm-5.1", "fallback_mode")
321
+
322
+ from backend.services.memory_service import GlobalOmniMemory
323
+ omni_context = GlobalOmniMemory.get_global_context_stream()
324
+
325
+ # Construct exact prompt with resume logic and hive mind context
326
+ actual_prompt = _build_resume_prompt(prompt, partial_so_far, _extract_last_word(partial_so_far))
327
+ if not partial_so_far:
328
+ # NVIDIA models take no separate system channel here, so the persona
329
+ # prompt is prepended (same place the hive-mind context goes).
330
+ _sys = _persona_system_prompt(persona, task_type, live_state, can_search, can_act, can_read)
331
+ actual_prompt = (_sys + "\n\n" if _sys else "") + omni_context + actual_prompt
332
+
333
+ # Delegate to the NVIDIA Vault API directly
334
+ from backend.services.nvidia_vault import call_nvidia_model, NvidiaModel
335
+ # GLM is the intended head of the heavy-compute fallback tier. call_nvidia_model
336
+ # already cascades from here through the rest of that tier (MiniMax, DeepSeek,
337
+ # Nemotron-Ultra) with up to 3 key-failovers per model, so a single slow or
338
+ # unavailable model does not strand the request.
339
+ #
340
+ # Only the model ID changed: "z-ai/glm-5.1" was retired upstream and returned
341
+ # HTTP 410 on all 15 keys, which wasted the first attempt of every heavy-tier
342
+ # call. It is now glm-5.2.
343
+ fallback_response = call_nvidia_model(actual_prompt, NvidiaModel.GLM_5_2)
344
+
345
+ # Record what NVIDIA just did to the Hive Mind
346
+ GlobalOmniMemory.record_action("NVIDIA", "glm-5.1", task_type, fallback_response[:100].replace('\n', ' ') + "...")
347
+
348
+ combined = partial_so_far + fallback_response
349
+ _mark_checkpoint_complete(task_id)
350
+ return combined
351
+
352
+ def _gemini_call_sync(prompt: str, task_id: str, task_type: str,
353
+ partial_so_far: str = "", persona: str = "jarvis",
354
+ live_state: Optional[dict] = None,
355
+ can_search: bool = False, can_act: bool = False,
356
+ can_read: bool = False) -> str:
357
+ key = _get_key_for_task(task_type)
358
+
359
+ # 1. If we know it's exhausted, fallback immediately
360
+ if key in EXHAUSTED_GOOGLE_KEYS:
361
+ logging.info(f"TokenManager: Google Key exhausted. Routing {task_type} instantly to NVIDIA Fallback.")
362
+ return _nvidia_fallback_call_sync(prompt, task_id, task_type, partial_so_far,
363
+ persona, live_state, can_search, can_act, can_read)
364
+
365
+ genai.configure(api_key=key)
366
+ _broadcast_live_key_status("GOOGLE", MODEL, "primary_active")
367
+
368
+ from backend.services.memory_service import GlobalOmniMemory
369
+ omni_context = GlobalOmniMemory.get_global_context_stream()
370
+
371
+ # Shape the reply in the active assistant's voice (JARVIS/FRIDAY). Skip on a
372
+ # resume continuation so we don't restate the persona mid-sentence.
373
+ _sys = (_persona_system_prompt(persona, task_type, live_state, can_search, can_act, can_read)
374
+ if not partial_so_far else None)
375
+ actual_prompt = _build_resume_prompt(prompt, partial_so_far, _extract_last_word(partial_so_far))
376
+ if not partial_so_far:
377
+ actual_prompt = omni_context + actual_prompt
378
+
379
+ # Walk the Gemini cascade before considering the NVIDIA tier. A model that is
380
+ # retired or momentarily at capacity must not cost the whole Google side —
381
+ # the next model in the list is usually healthy.
382
+ last_exc = None
383
+ for attempt_model in GEMINI_MODEL_CASCADE:
384
+ try:
385
+ model = (genai.GenerativeModel(attempt_model, system_instruction=_sys)
386
+ if _sys else genai.GenerativeModel(attempt_model))
387
+ response = model.generate_content(actual_prompt)
388
+ new_text = response.text or ""
389
+
390
+ if attempt_model != MODEL:
391
+ logging.warning("TokenManager: primary model unavailable; served by %s", attempt_model)
392
+ _broadcast_live_key_status("GOOGLE", attempt_model, "primary_active")
393
+
394
+ # Record what Google just did to the Hive Mind
395
+ GlobalOmniMemory.record_action("GOOGLE", attempt_model, task_type,
396
+ new_text[:100].replace('\n', ' ') + "...")
397
+
398
+ combined = partial_so_far + new_text
399
+ _mark_checkpoint_complete(task_id)
400
+ return combined
401
+ except Exception as exc:
402
+ last_exc = exc
403
+ if _is_transient_error(str(exc).lower()):
404
+ logging.info("TokenManager: %s unavailable (%s); trying next Gemini model.",
405
+ attempt_model, str(exc)[:70])
406
+ continue
407
+ raise
408
+
409
+ # Every Gemini model failed — fall through to the original handling, which
410
+ # decides between key-exhaustion bookkeeping and the NVIDIA tier.
411
+ try:
412
+ raise last_exc
413
+ except Exception as e:
414
+ err_str = str(e).lower()
415
+ # Any TRANSIENT upstream failure should fall back to NVIDIA, not just a
416
+ # quota error. This previously matched 429/quota only, so a Google-side
417
+ # capacity 503 ("This model is currently experiencing high demand" — which
418
+ # gemini-3.5-flash returns constantly) fell through to `raise` and killed
419
+ # the request outright, never reaching the NVIDIA tier that exists for
420
+ # exactly this situation.
421
+ is_rate_limit = (
422
+ "resourceexhausted" in err_str
423
+ or "429" in err_str
424
+ or "quota" in err_str
425
+ or "rate limit" in err_str
426
+ or "too many requests" in err_str
427
+ # Google-side capacity / availability, not our usage:
428
+ or "503" in err_str
429
+ or "500" in err_str
430
+ or "high demand" in err_str
431
+ or "overloaded" in err_str
432
+ or "unavailable" in err_str
433
+ or "internal error" in err_str
434
+ or "deadline" in err_str
435
+ or "timeout" in err_str
436
+ )
437
+ # A capacity/availability failure is the MODEL's problem, not the key's.
438
+ # Blacklisting the key for a 503 would permanently route a perfectly
439
+ # healthy credential to the fallback tier — so only real quota errors
440
+ # mark a key exhausted. Both still fall back to NVIDIA.
441
+ is_key_exhausted = (
442
+ "resourceexhausted" in err_str
443
+ or "429" in err_str
444
+ or "quota" in err_str
445
+ or "rate limit" in err_str
446
+ or "too many requests" in err_str
447
+ )
448
+ if is_rate_limit:
449
+ logging.warning(
450
+ "TokenManager: transient upstream failure for [%s:%s] (%s). Triggering NVIDIA Fallback.",
451
+ task_type, task_id, "quota" if is_key_exhausted else "capacity/503",
452
+ )
453
+ if not is_key_exhausted:
454
+ # Model-side outage: keep the key healthy, just serve this request
455
+ # from the fallback tier.
456
+ return _nvidia_fallback_call_sync(prompt, task_id, task_type, partial_so_far, persona)
457
+ # 2. Add to exhausted pool
458
+ if key not in EXHAUSTED_GOOGLE_KEYS:
459
+ EXHAUSTED_GOOGLE_KEYS.add(key)
460
+
461
+ # We can't spawn an async task cleanly from a sync thread without a loop,
462
+ # so we will raise a special exception that the async wrapper catches to spawn the pinger.
463
+ raise _Google429Trigger(key=key, prompt=prompt, task_id=task_id, task_type=task_type, partial_so_far=partial_so_far, persona=persona)
464
+
465
+ return _nvidia_fallback_call_sync(prompt, task_id, task_type, partial_so_far, persona)
466
+ raise
467
+
468
+ class _Google429Trigger(Exception):
469
+ def __init__(self, key, prompt, task_id, task_type, partial_so_far, persona):
470
+ self.key = key
471
+ self.prompt = prompt
472
+ self.task_id = task_id
473
+ self.task_type = task_type
474
+ self.partial_so_far = partial_so_far
475
+ self.persona = persona
476
+
477
+ TokenLimitHit = _Google429Trigger
478
+
479
+ async def gemini_call_with_checkpoint(
480
+ prompt: str,
481
+ task_type: str = "general",
482
+ persona: str = "jarvis",
483
+ task_id: Optional[str] = None,
484
+ partial_so_far: str = "",
485
+ live_state: Optional[dict] = None,
486
+ can_search: bool = False,
487
+ can_act: bool = False,
488
+ can_read: bool = False,
489
+ ) -> str:
490
+ """
491
+ Universal Gemini 3.5 Flash call with automatic checkpoint & resume.
492
+
493
+ `live_state` is the set of facts measured for this turn (is the PC actually
494
+ connected, what time is it, what is the device). It reaches the model as part
495
+ of the composed identity. Passing nothing is allowed and honest — the prompt
496
+ then tells him he has no measured facts — but a conversational caller that
497
+ can measure state and does not is how "connected, sir" gets said about a
498
+ machine that is switched off.
499
+ """
500
+ if task_id is None:
501
+ task_id = str(uuid.uuid4())
502
+
503
+ try:
504
+ return await asyncio.to_thread(
505
+ _gemini_call_sync, prompt, task_id, task_type, partial_so_far, persona,
506
+ live_state, can_search, can_act, can_read
507
+ )
508
+ except _Google429Trigger as e:
509
+ # 1. Start the token refresh monitor loop
510
+ asyncio.create_task(start_token_refresh_monitor())
511
+ # 2. Immediately complete THIS task using NVIDIA fallback
512
+ return await asyncio.to_thread(
513
+ _nvidia_fallback_call_sync, e.prompt, e.task_id, e.task_type, e.partial_so_far,
514
+ e.persona, live_state, can_search, can_act, can_read
515
+ )
516
+
517
+ # For backward compatibility where 'run_task' is imported from token_manager
518
+ async def run_task(prompt: str, task_type: str = "general", persona: str = "jarvis",
519
+ live_state: Optional[dict] = None, can_search: bool = False,
520
+ can_act: bool = False, can_read: bool = False) -> str:
521
+ return await gemini_call_with_checkpoint(prompt, task_type, persona,
522
+ live_state=live_state, can_search=can_search,
523
+ can_act=can_act, can_read=can_read)
524
+
525
+
526
+ # ─────────────────────────────────────────────────────────────────────────────
527
+ # Background Cloud Monitor — polls until tokens refresh then resumes
528
+ # ─────────────────────────────────────────────────────────────────────────────
529
+
530
+ _monitor_running = False
531
+
532
+ async def start_token_refresh_monitor():
533
+ """
534
+ Background loop that:
535
+ 1. Checks for pending_resume checkpoints every 60s
536
+ 2. Probes Gemini with a minimal token-cost ping
537
+ 3. When the probe succeeds, resumes ALL pending tasks from their exact last word
538
+ 4. Broadcasts omega:task_resumed WS event for each resumed task
539
+ """
540
+ global _monitor_running
541
+ if _monitor_running:
542
+ return
543
+ _monitor_running = True
544
+ logging.info("TokenManager: Token refresh monitor STARTED.")
545
+
546
+ while _monitor_running:
547
+ try:
548
+ pending = _get_pending_checkpoints()
549
+
550
+ # We must probe if there are pending checkpoints OR if we have exhausted keys that need refreshing
551
+ if pending or EXHAUSTED_GOOGLE_KEYS:
552
+ if pending:
553
+ logging.info(f"TokenManager: {len(pending)} checkpoint(s) pending resume. Probing API...")
554
+ else:
555
+ logging.info("TokenManager: No checkpoints, but EXHAUSTED_GOOGLE_KEYS has entries. Probing API...")
556
+
557
+ # Lightweight probe — minimal cost
558
+ # Check each exhausted key independently
559
+ keys_to_probe = list(EXHAUSTED_GOOGLE_KEYS) if EXHAUSTED_GOOGLE_KEYS else [_get_key_for_task("general")]
560
+ recovered_keys = []
561
+
562
+ for key in keys_to_probe:
563
+ try:
564
+ genai.configure(api_key=key)
565
+ model = genai.GenerativeModel(MODEL)
566
+ model.generate_content("ping")
567
+ recovered_keys.append(key)
568
+ except Exception as probe_err:
569
+ err_str = str(probe_err).lower()
570
+ if "resourceexhausted" in err_str or "429" in err_str or "quota" in err_str:
571
+ pass # Still exhausted
572
+ else:
573
+ logging.error(f"TokenManager: Probe error on key: {probe_err}")
574
+
575
+ if not recovered_keys and keys_to_probe:
576
+ logging.warning("TokenManager: API still exhausted. Will retry in 60s.")
577
+
578
+ if recovered_keys:
579
+ logging.info(f"TokenManager: API Probe OK for {len(recovered_keys)} keys. Clearing them from EXHAUSTED_GOOGLE_KEYS.")
580
+ for k in recovered_keys:
581
+ if k in EXHAUSTED_GOOGLE_KEYS:
582
+ EXHAUSTED_GOOGLE_KEYS.remove(k)
583
+
584
+ if pending:
585
+ # Resume all pending checkpoints now that some keys are refreshed
586
+ for checkpoint in pending:
587
+ asyncio.create_task(_resume_checkpoint(checkpoint))
588
+
589
+ except Exception as e:
590
+ logging.error(f"TokenManager: Monitor loop error: {e}")
591
+
592
+ await asyncio.sleep(60)
593
+
594
+ logging.info("TokenManager: Token refresh monitor STOPPED.")
595
+
596
+
597
+ async def _resume_checkpoint(checkpoint: dict):
598
+ """Resume a single checkpoint from its exact last word."""
599
+ task_id = checkpoint["id"]
600
+ task_type = checkpoint["task_type"]
601
+ original_prompt = checkpoint["original_prompt"]
602
+ partial = checkpoint["partial_result"] or ""
603
+ last_word = checkpoint["last_word"] or ""
604
+ persona = checkpoint.get("persona", "jarvis")
605
+
606
+ logging.info(f"TokenManager: Resuming [{task_type}:{task_id}] from last_word='{last_word}'")
607
+ _mark_checkpoint_resumed(task_id)
608
+
609
+ try:
610
+ # Broadcast WS notification
611
+ from backend.ws.agent_ws import ws_manager
612
+ await ws_manager.broadcast({
613
+ "event": "omega:task_resuming",
614
+ "payload": {
615
+ "task_id": task_id,
616
+ "task_type": task_type,
617
+ "last_word": last_word,
618
+ "message": (
619
+ f"Token limit has refreshed, sir. Resuming {task_type} task from exactly '{last_word}'..."
620
+ if persona == "jarvis" else
621
+ f"We're back boss! Tokens refreshed — resuming {task_type} from exactly '{last_word}'!"
622
+ )
623
+ }
624
+ })
625
+
626
+ # Re-run the Gemini call from exact checkpoint
627
+ result = await gemini_call_with_checkpoint(
628
+ prompt=original_prompt,
629
+ task_type=task_type,
630
+ persona=persona,
631
+ task_id=task_id,
632
+ partial_so_far=partial
633
+ )
634
+
635
+ _mark_checkpoint_complete(task_id)
636
+
637
+ # Route result back to its originating system
638
+ await _dispatch_resumed_result(task_type, task_id, result, persona, checkpoint)
639
+
640
+ await ws_manager.broadcast({
641
+ "event": "omega:task_resumed",
642
+ "payload": {
643
+ "task_id": task_id,
644
+ "task_type": task_type,
645
+ "message": (
646
+ f"{task_type.title()} task completed after token refresh, sir."
647
+ if persona == "jarvis" else
648
+ f"Done boss! {task_type.title()} task finished after the token refresh!"
649
+ )
650
+ }
651
+ })
652
+
653
+ except TokenLimitHit:
654
+ # Still throttled — will retry next monitor cycle
655
+ logging.warning(f"TokenManager: Still throttled on resume for [{task_id}]. Reverting to pending_resume.")
656
+ try:
657
+ with sqlite3.connect(get_db_path()) as conn:
658
+ conn.execute(
659
+ "UPDATE token_checkpoints SET status='pending_resume', retry_count=retry_count+1 WHERE id=?",
660
+ (task_id,)
661
+ )
662
+ conn.commit()
663
+ except Exception as e:
664
+ # No local `import logging` — it would shadow the module import for
665
+ # the whole of _resume_checkpoint() and break the logging calls above.
666
+ logging.getLogger(__name__).error(f"Swallowed exception: {e}")
667
+ except Exception as e:
668
+ logging.error(f"TokenManager: Resume error for [{task_id}]: {e}")
669
+
670
+
671
+ async def _dispatch_resumed_result(task_type: str, task_id: str,
672
+ result: str, persona: str, checkpoint: dict):
673
+ """Route the completed result back to the correct subsystem."""
674
+ try:
675
+ if task_type == "research":
676
+ # Re-parse the JSON research note and save it
677
+ import json
678
+ from backend.omega.research_engine import ResearchNote, ResearchCategory, _save_research_note
679
+ raw = result.strip().strip("```json").strip("```").strip()
680
+ data = json.loads(raw)
681
+ raw_category = data.get("category", "Research Notes")
682
+ valid_categories = {c.value: c for c in ResearchCategory}
683
+ category = valid_categories.get(raw_category, ResearchCategory.RESEARCH_NOTES)
684
+ note = ResearchNote(
685
+ title=data.get("title", "Resumed Research"),
686
+ summary=data.get("summary", result[:500]),
687
+ importance=data.get("importance", "MEDIUM"),
688
+ category=category,
689
+ source=data.get("source", "Gemini 3.5 Flash (resumed)"),
690
+ recommended_action=data.get("recommended_action", "")
691
+ )
692
+ _save_research_note(note)
693
+ logging.info(f"TokenManager: Research note saved after resume: {note.title}")
694
+
695
+ elif task_type == "implement":
696
+ # Re-parse code blocks and hot-reload them
697
+ from backend.omega.auto_upgrade import parse_code_blocks, write_and_hot_reload, log_upgrade_to_db
698
+ code_blocks = parse_code_blocks(result)
699
+ files_modified = []
700
+ for block in code_blocks:
701
+ await write_and_hot_reload(block.filepath, block.code)
702
+ files_modified.append(block.filepath)
703
+ log_upgrade_to_db(
704
+ checkpoint.get("original_prompt", "Resumed task")[:60],
705
+ files_modified, persona, "success_after_resume"
706
+ )
707
+ logging.info(f"TokenManager: Implementation resumed and hot-reloaded: {files_modified}")
708
+
709
+ else:
710
+ # For gate/captcha/monitor tasks, just log the full result
711
+ logging.info(f"TokenManager: [{task_type}] resumed result stored (length={len(result)})")
712
+
713
+ except Exception as e:
714
+ logging.error(f"TokenManager: dispatch_resumed_result error for [{task_type}]: {e}")
715
+
716
+
717
+ def stop_token_refresh_monitor():
718
+ global _monitor_running
719
+ _monitor_running = False
720
+ logging.info("TokenManager: Token refresh monitor signalled to stop.")
backend/services/usb_monitor.py CHANGED
@@ -112,7 +112,9 @@ def get_active_usb_drives():
112
  "drive_letter": drive_letter
113
  })
114
  except Exception as e:
115
- import logging
 
 
116
  logging.error(f"Error enumerating active USB drives: {e}")
117
  return drives
118
 
@@ -184,7 +186,25 @@ async def start_usb_monitor():
184
  break
185
  if is_pendrive: break
186
  except Exception as e:
187
- import logging; logging.getLogger(__name__).error(f"Swallowed exception: {e}")
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
188
 
189
  if is_pendrive:
190
  try:
 
112
  "drive_letter": drive_letter
113
  })
114
  except Exception as e:
115
+ # Module-scope `logging`, not a local import — the same shadowing trap
116
+ # that broke fetch_pyusb. Harmless here today only because nothing reads
117
+ # `logging` earlier in this function; that is luck, not design.
118
  logging.error(f"Error enumerating active USB drives: {e}")
119
  return drives
120
 
 
186
  break
187
  if is_pendrive: break
188
  except Exception as e:
189
+ # NO local `import logging` here.
190
+ #
191
+ # It was the cause of the error that filled the log
192
+ # every two seconds. A local import binds the name
193
+ # for the WHOLE function, so `logging` became a
194
+ # local of fetch_pyusb — and the outer handler
195
+ # below, which runs when usb.core.find() fails,
196
+ # read it before this line had ever executed:
197
+ #
198
+ # UnboundLocalError: cannot access local variable
199
+ # 'logging' where it is not associated with a value
200
+ #
201
+ # The handler meant to REPORT the PyUSB failure was
202
+ # therefore the thing that crashed, replacing the
203
+ # real reason with a misleading one. `logging` is
204
+ # already imported at module scope; using it is all
205
+ # that was ever needed.
206
+ logging.getLogger(__name__).error(
207
+ "USB interface probe failed: %s", e)
208
 
209
  if is_pendrive:
210
  try:
backend/tools/web_search_tools.py CHANGED
@@ -217,10 +217,34 @@ async def search_web(query: str, num_results: int = 5) -> dict:
217
  # worked if it actually returned results; an empty 200 falls through, so a
218
  # rate-limited or misconfigured primary degrades instead of silently
219
  # returning nothing to the assistant.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
220
  for provider, available in ((_brave_search, brave_api_key),
221
  (_serpapi_search, serpapi_key),
222
- # Keyless and unthrottled the reason research
223
- # keeps working with no credentials configured.
 
 
224
  (_wikipedia_search, True)):
225
  if not available:
226
  continue
@@ -229,10 +253,7 @@ async def search_web(query: str, num_results: int = 5) -> dict:
229
  return out
230
  logging.warning("web search provider %s failed (%s); trying next",
231
  provider.__name__, out.get("error"))
232
- # Keyless last resort. Unlike the old instant-answer endpoint this returns
233
- # real ranked results, so search/OSINT/threat-modelling still function with
234
- # no credential configured at all.
235
- return await asyncio.to_thread(_duck_html_search)
236
 
237
  async def fetch_page(url: str) -> dict:
238
  """Fetch page via Playwright, bypassing CAPTCHAs if encountered"""
 
217
  # worked if it actually returned results; an empty 200 falls through, so a
218
  # rate-limited or misconfigured primary degrades instead of silently
219
  # returning nothing to the assistant.
220
+ #
221
+ # WIKIPEDIA IS LAST, NOT THIRD.
222
+ #
223
+ # It used to sit above the web scrape, and because it is always "available"
224
+ # it answered nearly every keyless search — which is how OSINT came back
225
+ # confidently wrong. Measured, before this change:
226
+ #
227
+ # "Sundar Pichai" -> 4 Wikipedia articles (Sundar Pichai, Google Gemini,
228
+ # Alphabet Inc., Google). The live web was never asked.
229
+ # "who is Elon Musk" -> "Wealth of Elon Musk", "Errol Musk", "Trump-Musk
230
+ # feud" — an encyclopedia's idea of relevance.
231
+ #
232
+ # An encyclopedia cannot answer the questions this assistant is actually
233
+ # asked: find a specific syllabus PDF, find a person who has a profile but
234
+ # no article, find anything that happened recently. It returned *something*
235
+ # plausible for every query, so the fallback chain never advanced to a real
236
+ # search engine, and the model then wrote an answer out of whatever loosely
237
+ # related articles it was handed. Junk results are worse than no results:
238
+ # empty invites "I couldn't find it", plausible-but-wrong invites invention.
239
+ #
240
+ # Below the scrape it still does its real job — the thing that keeps research
241
+ # alive when DuckDuckGo throws its anti-automation challenge.
242
  for provider, available in ((_brave_search, brave_api_key),
243
  (_serpapi_search, serpapi_key),
244
+ # Keyless, and actual ranked web results.
245
+ (_duck_html_search, True),
246
+ # Keyless and unthrottled: the safety net for when
247
+ # the scrape above is challenged, not the default.
248
  (_wikipedia_search, True)):
249
  if not available:
250
  continue
 
253
  return out
254
  logging.warning("web search provider %s failed (%s); trying next",
255
  provider.__name__, out.get("error"))
256
+ return {"error": "no search provider returned results"}
 
 
 
257
 
258
  async def fetch_page(url: str) -> dict:
259
  """Fetch page via Playwright, bypassing CAPTCHAs if encountered"""
backend/voice/engines/kokoro_engine.py CHANGED
The diff for this file is too large to render. See raw diff
 
backend/voice/jarvis_live.py ADDED
@@ -0,0 +1,202 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """The JARVIS voice for everything JARVIS actually says.
2
+
3
+ `backend/voice/tts.py` already routes the *backend's* JARVIS speech through the
4
+ Kokoro+RVC pipeline. But the desktop assistant does not speak through that
5
+ path at all — tray.py, modules/executor.py, modules/hud.py, the automation
6
+ triggers, the Gmail connector and the tour all call `core.voice.speak()`, which
7
+ synthesises with Edge-TTS. So the tuned voice was reachable from the API and
8
+ from the phone, and from nowhere the user actually hears day to day.
9
+
10
+ This module is the bridge. It renders a line with the real pipeline and hands
11
+ back a WAV path that `core.voice`'s existing player can use unchanged.
12
+
13
+ Three things make that practical rather than merely possible:
14
+
15
+ * A DISK CACHE. Assistants repeat themselves — "Done.", "Yes, sir.",
16
+ "Standing by." A repeated line costs a file read instead of seconds of
17
+ inference, which is what makes short acknowledgements usable at all.
18
+
19
+ * A WARM-UP. The first render pays ~35 s of model loading. Left alone, the
20
+ first thing JARVIS says after boot arrives half a minute late. `warm_up()`
21
+ moves that cost to startup, where nothing is waiting on it.
22
+
23
+ * A FALLBACK THAT NEVER MUTES HIM. If torch, RVC or the model files are
24
+ missing — a stripped exe build, a fresh machine — this returns None and the
25
+ caller keeps its Edge-TTS path. A degraded voice is recoverable; silence
26
+ is not.
27
+ """
28
+
29
+ from __future__ import annotations
30
+
31
+ import hashlib
32
+ import logging
33
+ import os
34
+ import threading
35
+ import time
36
+
37
+ logger = logging.getLogger(__name__)
38
+
39
+ # ── Configuration ────────────────────────────────────────────────────────────
40
+
41
+ # The music bed is OFF for live speech, and that is deliberate.
42
+ #
43
+ # The bed exists to match the reference recording, which is a scored piece of
44
+ # film audio. Under a 40-second demo it is the thing that made the clip sound
45
+ # like the real JARVIS. Under every "Done." and every alert, all day, a looping
46
+ # 40-second cue would be exhausting — and each utterance would restart the loop
47
+ # from the same point, so the same few bars would play over and over.
48
+ #
49
+ # Set to 5.0 to get the demo's bed back; nothing else needs to change.
50
+ LIVE_AMBIENCE_LEVEL = 0.0
51
+
52
+ # Bump when the voice itself changes, so old cache entries are not reused for
53
+ # a voice that no longer exists. This is a cache key component, not a version
54
+ # number anyone reads.
55
+ VOICE_REVISION = "2026-08-11-a"
56
+
57
+ # Lines longer than this are not worth caching — they are one-off sentences
58
+ # that will never repeat, and caching them only fills the disk.
59
+ CACHE_MAX_CHARS = 240
60
+ CACHE_MAX_FILES = 400
61
+
62
+ _lock = threading.Lock()
63
+ _warm_thread: threading.Thread | None = None
64
+ _available: bool | None = None # None = not yet determined
65
+
66
+
67
+ def _cache_dir() -> str:
68
+ base = os.environ.get("JARVIS_APP_DATA_DIR") or os.environ.get("LOCALAPPDATA") \
69
+ or os.path.expanduser("~")
70
+ d = os.path.join(base, "jarvis_voice_cache")
71
+ os.makedirs(d, exist_ok=True)
72
+ return d
73
+
74
+
75
+ def _key(text: str, profile: str) -> str:
76
+ raw = f"{VOICE_REVISION}|{profile}|{LIVE_AMBIENCE_LEVEL}|{text}"
77
+ return hashlib.sha256(raw.encode("utf-8")).hexdigest()[:32]
78
+
79
+
80
+ def is_available() -> bool:
81
+ """Can the real pipeline run here?
82
+
83
+ Answered by importing, not by looking for files: the failure modes that
84
+ matter (no torch, no CUDA runtime, a broken faiss/numpy pairing) are all
85
+ import-time, and every one of them has silently degraded this pipeline to
86
+ plain Kokoro before.
87
+ """
88
+ global _available
89
+ if _available is not None:
90
+ return _available
91
+ try:
92
+ import backend.voice.engines.kokoro_engine as ke # noqa: F401
93
+ _available = True
94
+ except Exception as exc:
95
+ logger.warning("[JARVIS voice] pipeline unavailable (%s); "
96
+ "falling back to Edge-TTS.", exc)
97
+ _available = False
98
+ return _available
99
+
100
+
101
+ def warm_up() -> None:
102
+ """Load the models in the background so the first line is not the slow one.
103
+
104
+ Idempotent and non-blocking. Safe to call from startup paths that must not
105
+ wait for anything.
106
+ """
107
+ global _warm_thread
108
+ with _lock:
109
+ if _warm_thread is not None:
110
+ return
111
+
112
+ def _run():
113
+ try:
114
+ t0 = time.time()
115
+ import backend.voice.engines.kokoro_engine as ke
116
+ ke._get_kokoro()
117
+ ke._get_rvc()
118
+ logger.info("[JARVIS voice] models warm in %.1fs", time.time() - t0)
119
+ except Exception as exc:
120
+ logger.warning("[JARVIS voice] warm-up failed: %s", exc)
121
+
122
+ _warm_thread = threading.Thread(target=_run, daemon=True,
123
+ name="jarvis-voice-warmup")
124
+ _warm_thread.start()
125
+
126
+
127
+ def _prune_cache(d: str) -> None:
128
+ """Keep the cache bounded, oldest-accessed first."""
129
+ try:
130
+ files = [os.path.join(d, f) for f in os.listdir(d) if f.endswith(".wav")]
131
+ if len(files) <= CACHE_MAX_FILES:
132
+ return
133
+ files.sort(key=lambda p: os.path.getatime(p))
134
+ for p in files[:len(files) - CACHE_MAX_FILES]:
135
+ try:
136
+ os.remove(p)
137
+ except OSError:
138
+ pass
139
+ except Exception:
140
+ pass
141
+
142
+
143
+ def render_to_file(text: str, profile: str | None = None) -> str | None:
144
+ """Render `text` in the JARVIS voice; return a WAV path, or None.
145
+
146
+ None means "use your existing fallback" — it is never an error the caller
147
+ has to handle beyond that.
148
+ """
149
+ text = (text or "").strip()
150
+ if not text or not is_available():
151
+ return None
152
+
153
+ try:
154
+ import backend.voice.engines.kokoro_engine as ke
155
+ except Exception:
156
+ return None
157
+
158
+ name = profile or getattr(ke, "ACTIVE_PROFILE", "m4a")
159
+ cacheable = len(text) <= CACHE_MAX_CHARS
160
+ d = _cache_dir()
161
+ path = os.path.join(d, _key(text, name) + ".wav")
162
+
163
+ if cacheable and os.path.exists(path) and os.path.getsize(path) > 1024:
164
+ try:
165
+ os.utime(path, None) # keep it fresh for the pruner
166
+ except OSError:
167
+ pass
168
+ return path
169
+
170
+ try:
171
+ import asyncio
172
+
173
+ # The bed level is a module global on the engine, and the engine has no
174
+ # per-call override for it. Set it around the call and put it back, so
175
+ # a caller that renders WITH a bed (the demo scripts) is unaffected.
176
+ prev = getattr(ke, "AMBIENCE_LEVEL", 0.0)
177
+ ke.AMBIENCE_LEVEL = LIVE_AMBIENCE_LEVEL
178
+ try:
179
+ t0 = time.time()
180
+ data = asyncio.run(ke.synthesize_kokoro(text, profile=name))
181
+ finally:
182
+ ke.AMBIENCE_LEVEL = prev
183
+
184
+ if not data or len(data) < 1024:
185
+ logger.warning("[JARVIS voice] empty render for %r", text[:40])
186
+ return None
187
+
188
+ # Write to a temp name and rename, so a cache hit can never land on a
189
+ # half-written file if two threads race on the same line.
190
+ tmp = path + f".{os.getpid()}.{threading.get_ident()}.part"
191
+ with open(tmp, "wb") as fh:
192
+ fh.write(data)
193
+ os.replace(tmp, path)
194
+
195
+ logger.info("[JARVIS voice] rendered %d chars in %.2fs", len(text),
196
+ time.time() - t0)
197
+ if cacheable:
198
+ _prune_cache(d)
199
+ return path
200
+ except Exception as exc:
201
+ logger.warning("[JARVIS voice] render failed (%s); falling back.", exc)
202
+ return None
backend/voice/tts.py CHANGED
@@ -41,6 +41,18 @@ class TTSPipeline:
41
  """
42
  import logging
43
 
 
 
 
 
 
 
 
 
 
 
 
 
44
  # ── FRIDAY path — completely untouched, existing logic, zero changes ──
45
  if personality != "jarvis":
46
  return await self._synthesize_friday(text, personality, language)
 
41
  """
42
  import logging
43
 
44
+ # Numbers become words HERE, before the engine split.
45
+ #
46
+ # This lived inside kokoro_engine, which meant only the JARVIS English
47
+ # path ever got it — FRIDAY, the Edge-TTS fallback and every non-English
48
+ # language still read "15,444" as "fifteen" then "zero zero zero",
49
+ # because the comma splits the token and the rest is voiced digit by
50
+ # digit. This is the one place every engine passes through, so it is the
51
+ # only place the fix belongs. Applying it twice is harmless: after the
52
+ # first pass there are no digits left to match.
53
+ from backend.voice.engines.kokoro_engine import speak_numbers
54
+ text = speak_numbers(text)
55
+
56
  # ── FRIDAY path — completely untouched, existing logic, zero changes ──
57
  if personality != "jarvis":
58
  return await self._synthesize_friday(text, personality, language)
config.py CHANGED
@@ -433,8 +433,78 @@ ANALYTICS_DIR = os.path.join(COPIES_DIR, "analytics")
433
  REPAIR_DIR = os.path.join(COPIES_DIR, "repair")
434
 
435
  # API
436
- GEMINI_API_KEY = "" # Set your Gemini API key here
437
- GEMINI_MODEL = "gemini-2.5-flash-preview-04-17"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
438
 
439
  # Audio / Voice
440
  # FRIDAY OMEGA - Young, Natural Human Voice (20-25 female)
 
433
  REPAIR_DIR = os.path.join(COPIES_DIR, "repair")
434
 
435
  # API
436
+ #
437
+ # This was the literal `GEMINI_API_KEY = ""` with a note to paste a key in, and
438
+ # nothing ever read .env — which has held a valid key the whole time. The cost
439
+ # was not a warning in a log. core/brain.py imports this name, _ask_gemini
440
+ # raised on every single call, and core/brain_router fell through to the
441
+ # provider named "local" — which is NOT a local model, just canned strings from
442
+ # _local_fallback(). So every "reply" from JARVIS or FRIDAY on the desktop was
443
+ # stock filler, including the "Got it: <x>. Tell me the outcome you want." that
444
+ # core/commands.py has a comment complaining about.
445
+ #
446
+ # Environment first (a Space secret or a shell export must win), then .env, then
447
+ # the literal, so a real deployment can override without editing source.
448
+ def _read_env_file(name: str) -> str:
449
+ try:
450
+ env_path = Path(__file__).resolve().parent / ".env"
451
+ if not env_path.exists():
452
+ return ""
453
+ for raw in env_path.read_text(encoding="utf-8", errors="replace").splitlines():
454
+ line = raw.strip()
455
+ if not line or line.startswith("#") or "=" not in line:
456
+ continue
457
+ key, _, value = line.partition("=")
458
+ if key.strip() == name:
459
+ return value.strip().strip('"').strip("'")
460
+ except Exception:
461
+ pass
462
+ return ""
463
+
464
+
465
+ GEMINI_API_KEY = os.environ.get("GEMINI_API_KEY") or _read_env_file("GEMINI_API_KEY") or ""
466
+ # Two failure modes have to be avoided here, and they pull in opposite
467
+ # directions.
468
+ #
469
+ # The original value was "gemini-2.5-flash-preview-04-17" — a dated preview that
470
+ # has since been RETIRED and answered every call with HTTP 404. Pinning a
471
+ # preview is a time bomb.
472
+ #
473
+ # The obvious fix, "gemini-flash-latest", is worse for this system: it resolves
474
+ # to whatever is newest, and the newest model carries the TIGHTEST free-tier
475
+ # quota. Measured: gemini-3.6-flash allows 20 requests PER DAY. An always-on
476
+ # assistant exhausts that before lunch and then runs on canned fallback strings
477
+ # for the rest of the day, silently, because brain_router swallows the 429.
478
+ #
479
+ # So: a stable, non-preview model with a workable free allowance, plus a
480
+ # fallback chain for when its quota is gone. Capability matters less than being
481
+ # answered at all — a slightly older model that replies beats a newer one that
482
+ # 429s.
483
+ GEMINI_MODEL = (os.environ.get("GEMINI_MODEL")
484
+ or _read_env_file("GEMINI_MODEL")
485
+ or "gemini-2.5-flash")
486
+
487
+ # Tried in order when the primary is rate-limited. Lite models have materially
488
+ # larger free quotas and are perfectly adequate for planning and judgement.
489
+ # Verified against list_models AND a real generateContent call on 2026-08-13.
490
+ # Three of the previous five were unusable: gemini-2.0-flash and
491
+ # gemini-2.0-flash-lite are gone entirely, and gemini-2.5-flash-lite is listed as
492
+ # available yet answers 404 "no longer available to new users" when called. A
493
+ # dead entry is not free — the cascade spends an attempt, a round trip and a
494
+ # retry on each one before reaching a model that works, which is why failover
495
+ # felt like an outage rather than a fallback.
496
+ #
497
+ # Verify the same way when this changes: listing a model does not prove the
498
+ # account may call it.
499
+ GEMINI_MODEL_FALLBACKS = [
500
+ "gemini-2.5-flash", # primary; 5 req/min on free tier
501
+ "gemini-3.5-flash-lite",
502
+ "gemini-3.1-flash-lite",
503
+ "gemini-3.5-flash",
504
+ # Deliberately last: gemini-3.6-flash is capped at ~20 requests per DAY on
505
+ # the free tier, so promoting it silently kills the brain by lunchtime.
506
+ "gemini-3.6-flash",
507
+ ]
508
 
509
  # Audio / Voice
510
  # FRIDAY OMEGA - Young, Natural Human Voice (20-25 female)
modules/alert_manager.py ADDED
@@ -0,0 +1,307 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Alerts that behave like a person raising something, not a loop shouting.
2
+
3
+ The watch loops in OMEGA announce a condition every time they observe it. A
4
+ condition that persists — a disk filling, a device still disconnected — is
5
+ therefore announced again on every pass, forever, at whatever interval that
6
+ loop happens to run. The existing `cooldown_s` in core.voice only spaces the
7
+ repeats out; it never stops them, and it is per-process, so two components
8
+ watching the same thing each get their own cooldown.
9
+
10
+ What is wanted instead:
11
+
12
+ * say it once;
13
+ * if it is not acknowledged, raise it again after a short while, because
14
+ something genuinely unattended should not be dropped silently;
15
+ * back off each time rather than nagging at a fixed rate;
16
+ * stop entirely once acknowledged, or once told to wait, or once muted;
17
+ * accept "not now", "tell me in 30 minutes" and "don't alert me about this
18
+ for two hours" as the way to control all of that.
19
+
20
+ State is on disk, keyed by TOPIC rather than by message text, so a rephrased
21
+ message about the same subject is still the same alert — "disk 91% full" and
22
+ "disk 93% full" must not read as two different things to nag about.
23
+ """
24
+
25
+ from __future__ import annotations
26
+
27
+ import json
28
+ import os
29
+ import re
30
+ import threading
31
+ import time
32
+ from typing import Optional
33
+
34
+ _STATE_DIR = os.environ.get("JARVIS_APP_DATA_DIR") or os.path.join(
35
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".friday_data")
36
+ _STATE_PATH = os.path.join(_STATE_DIR, "alert_state.json")
37
+
38
+ _lock = threading.RLock()
39
+
40
+ # Escalation ladder for an unacknowledged alert, in seconds.
41
+ #
42
+ # The first repeat is the 5 minutes asked for. After that it doubles and then
43
+ # stops: three unanswered mentions is the point at which repeating again is
44
+ # nagging rather than diligence. A genuinely critical condition should be
45
+ # raised with importance="critical", which uses the longer ladder below and
46
+ # does not give up.
47
+ REPEAT_LADDER = (300, 900, 2700)
48
+ CRITICAL_LADDER = (120, 300, 900, 1800, 3600)
49
+
50
+ # How long an alert stays "open" waiting to be acknowledged before it is
51
+ # considered stale and dropped.
52
+ MAX_ALERT_AGE = 24 * 3600
53
+
54
+
55
+ def _load() -> dict:
56
+ try:
57
+ with open(_STATE_PATH, "r", encoding="utf-8") as fh:
58
+ data = json.load(fh)
59
+ return data if isinstance(data, dict) else {}
60
+ except Exception:
61
+ return {}
62
+
63
+
64
+ def _save(data: dict) -> None:
65
+ try:
66
+ os.makedirs(_STATE_DIR, exist_ok=True)
67
+ tmp = _STATE_PATH + ".tmp"
68
+ with open(tmp, "w", encoding="utf-8") as fh:
69
+ json.dump(data, fh, indent=2)
70
+ os.replace(tmp, _STATE_PATH)
71
+ except Exception:
72
+ pass
73
+
74
+
75
+ def _topics(data: dict) -> dict:
76
+ return data.setdefault("topics", {})
77
+
78
+
79
+ # ── Total silence ───────────────────────────────────────────────────────────
80
+ #
81
+ # Distinct from mute(topic, seconds) above, which is per-subject and expires.
82
+ # "Mute" said out loud does not mean "quieten one topic for a while" — it means
83
+ # say nothing at all until told otherwise. There was no such state anywhere in
84
+ # the system: saying "mute" hit core.commands' instant_commands table and muted
85
+ # the WINDOWS MIXER, so JARVIS carried on speaking into a silenced output and
86
+ # came back at full volume the moment anything else unmuted the device.
87
+ #
88
+ # This latch has no timeout by design. An indefinite silence that expires on
89
+ # its own is not the thing that was asked for.
90
+
91
+ def silence_all(reason: str = "") -> None:
92
+ with _lock:
93
+ data = _load()
94
+ data["silenced"] = {"since": time.time(), "reason": reason or "asked"}
95
+ _save(data)
96
+
97
+
98
+ def unsilence_all() -> None:
99
+ with _lock:
100
+ data = _load()
101
+ data.pop("silenced", None)
102
+ _save(data)
103
+
104
+
105
+ def is_silenced() -> bool:
106
+ try:
107
+ return bool(_load().get("silenced"))
108
+ except Exception:
109
+ return False
110
+
111
+
112
+ def silence_info() -> Optional[dict]:
113
+ try:
114
+ got = _load().get("silenced")
115
+ return got if isinstance(got, dict) else None
116
+ except Exception:
117
+ return None
118
+
119
+
120
+ def should_speak(topic: str, importance: str = "normal") -> bool:
121
+ """May this topic be spoken right now?
122
+
123
+ Called by the raising code BEFORE it speaks. Returns False when the topic
124
+ is muted, snoozed, already acknowledged, or simply not due for its next
125
+ repeat yet — which is what turns a per-pass watch loop into an alert that
126
+ is mentioned a sensible number of times.
127
+ """
128
+ now = time.time()
129
+ with _lock:
130
+ data = _load()
131
+ t = _topics(data).get(topic)
132
+
133
+ if t is None:
134
+ _topics(data)[topic] = {
135
+ "first": now, "last": now, "count": 1,
136
+ "acknowledged": False, "muted_until": 0, "snoozed_until": 0,
137
+ "importance": importance,
138
+ }
139
+ _save(data)
140
+ return True
141
+
142
+ if t.get("muted_until", 0) > now:
143
+ return False
144
+ if t.get("snoozed_until", 0) > now:
145
+ return False
146
+ if t.get("acknowledged"):
147
+ return False
148
+ if now - t.get("first", now) > MAX_ALERT_AGE:
149
+ return False
150
+
151
+ ladder = CRITICAL_LADDER if t.get("importance") == "critical" else REPEAT_LADDER
152
+ count = int(t.get("count", 1))
153
+ if count > len(ladder):
154
+ return False # said enough; wait to be asked
155
+ due = t.get("last", 0) + ladder[count - 1]
156
+ if now < due:
157
+ return False
158
+
159
+ t["last"] = now
160
+ t["count"] = count + 1
161
+ _save(data)
162
+ return True
163
+
164
+
165
+ def acknowledge(topic: str) -> None:
166
+ """The user has dealt with it (or heard it). Stop raising it."""
167
+ with _lock:
168
+ data = _load()
169
+ t = _topics(data).setdefault(topic, {})
170
+ t["acknowledged"] = True
171
+ t["last"] = time.time()
172
+ _save(data)
173
+
174
+
175
+ def snooze(topic: str, seconds: float) -> None:
176
+ """Raise it again after `seconds`, then resume the normal ladder."""
177
+ with _lock:
178
+ data = _load()
179
+ t = _topics(data).setdefault(topic, {})
180
+ t["snoozed_until"] = time.time() + max(0.0, seconds)
181
+ t["acknowledged"] = False
182
+ # The repeat count resets: after an explicit "tell me in 30 minutes",
183
+ # the next mention is a fresh first mention, not a continuation of an
184
+ # escalation the user already responded to.
185
+ t["count"] = 1
186
+ _save(data)
187
+
188
+
189
+ def mute(topic: str, seconds: float) -> None:
190
+ """Say nothing about this topic at all for `seconds`."""
191
+ with _lock:
192
+ data = _load()
193
+ t = _topics(data).setdefault(topic, {})
194
+ t["muted_until"] = time.time() + max(0.0, seconds)
195
+ _save(data)
196
+
197
+
198
+ def reset(topic: str) -> None:
199
+ """The condition cleared. Forget it, so a recurrence starts fresh."""
200
+ with _lock:
201
+ data = _load()
202
+ _topics(data).pop(topic, None)
203
+ _save(data)
204
+
205
+
206
+ def open_topics() -> list[dict]:
207
+ """Alerts still waiting to be acknowledged — for 'what did I miss?'."""
208
+ now = time.time()
209
+ with _lock:
210
+ data = _load()
211
+ out = []
212
+ for name, t in _topics(data).items():
213
+ if t.get("acknowledged") or t.get("muted_until", 0) > now:
214
+ continue
215
+ out.append({"topic": name, "count": t.get("count", 1),
216
+ "first": t.get("first", 0),
217
+ "snoozed_until": t.get("snoozed_until", 0)})
218
+ return sorted(out, key=lambda d: d["first"])
219
+
220
+
221
+ # ── Natural language ─────────────────────────────────────────────────────────
222
+
223
+ _UNITS = {
224
+ "sec": 1, "secs": 1, "second": 1, "seconds": 1,
225
+ "min": 60, "mins": 60, "minute": 60, "minutes": 60,
226
+ "hr": 3600, "hrs": 3600, "hour": 3600, "hours": 3600,
227
+ "day": 86400, "days": 86400,
228
+ }
229
+
230
+ _WORD_NUMBERS = {
231
+ "a": 1, "an": 1, "one": 1, "two": 2, "three": 3, "four": 4, "five": 5,
232
+ "six": 6, "seven": 7, "eight": 8, "nine": 9, "ten": 10, "fifteen": 15,
233
+ "twenty": 20, "thirty": 30, "forty": 40, "forty-five": 45, "sixty": 60,
234
+ "half": 0.5,
235
+ }
236
+
237
+
238
+ def parse_duration(text: str) -> Optional[float]:
239
+ """Seconds from 'in 30 minutes', 'for 2 hours', 'half an hour'."""
240
+ lo = (text or "").lower()
241
+
242
+ m = re.search(r"half\s+an?\s+hour", lo)
243
+ if m:
244
+ return 1800.0
245
+
246
+ m = re.search(r"(\d+(?:\.\d+)?)\s*(" + "|".join(_UNITS) + r")\b", lo)
247
+ if m:
248
+ return float(m.group(1)) * _UNITS[m.group(2)]
249
+
250
+ words = "|".join(re.escape(w) for w in _WORD_NUMBERS)
251
+ m = re.search(r"\b(" + words + r")\s+(" + "|".join(_UNITS) + r")\b", lo)
252
+ if m:
253
+ return float(_WORD_NUMBERS[m.group(1)]) * _UNITS[m.group(2)]
254
+ return None
255
+
256
+
257
+ def parse_alert_command(text: str) -> Optional[dict]:
258
+ """Turn a spoken instruction into an action on the open alerts.
259
+
260
+ Returns {"action": ..., "seconds": ..., "topic": ...} or None when the
261
+ sentence is not about alerts at all. `topic` is None for "this"/"that",
262
+ meaning the most recently raised alert.
263
+
264
+ Deliberately conservative: an unrecognised sentence returns None so it can
265
+ be handled as an ordinary command, rather than being swallowed here.
266
+ """
267
+ lo = (text or "").strip().lower()
268
+ if not lo:
269
+ return None
270
+
271
+ secs = parse_duration(lo)
272
+
273
+ # "don't alert/tell/remind me about X for 2 hours"
274
+ if re.search(r"\b(don'?t|do not|stop|no more|quit)\b", lo) and \
275
+ re.search(r"\b(alert|alerts|tell|telling|remind|reminding|notify|"
276
+ r"notifying|warn|warning|mention)\b", lo):
277
+ topic = _extract_topic(lo)
278
+ return {"action": "mute", "seconds": secs or 3600.0, "topic": topic}
279
+
280
+ # "tell me in 30 minutes", "remind me in an hour", "not now"
281
+ if re.search(r"\b(later|not now|not right now|in a (bit|moment|while)|"
282
+ r"remind me|tell me|come back|ask me)\b", lo):
283
+ return {"action": "snooze", "seconds": secs or 1800.0,
284
+ "topic": _extract_topic(lo)}
285
+
286
+ # "ok", "got it", "noted", "I know", "acknowledged"
287
+ if re.fullmatch(r"(ok(ay)?|got it|noted|i know|understood|acknowledged|"
288
+ r"thanks|thank you|fine|sure|dismiss(ed)?)[.! ]*", lo):
289
+ return {"action": "acknowledge", "seconds": None, "topic": None}
290
+
291
+ # "what did I miss", "any alerts"
292
+ if re.search(r"\b(what did i miss|anything pending|any alerts|"
293
+ r"what'?s (pending|open|waiting))\b", lo):
294
+ return {"action": "list", "seconds": None, "topic": None}
295
+
296
+ return None
297
+
298
+
299
+ def _extract_topic(text: str) -> Optional[str]:
300
+ """Pull the subject out of 'about the disk space' style phrasing."""
301
+ m = re.search(r"\babout (?:the |my |this |that )?([a-z0-9 _-]{2,40})", text)
302
+ if not m:
303
+ return None
304
+ topic = m.group(1).strip()
305
+ # Trim a trailing duration clause: "about the disk for two hours".
306
+ topic = re.split(r"\b(for|in|until|till)\b", topic)[0].strip()
307
+ return topic or None
modules/assistant_identity.py CHANGED
@@ -1,6 +1,6 @@
1
  import json
2
  import os
3
- from typing import Dict
4
 
5
  from config import DATA_DIR
6
 
@@ -294,6 +294,414 @@ def get_identity_persona_prompt() -> str:
294
  return base + _favorites_block(m)
295
 
296
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
297
  def get_voice_id_for_mode(mode: str = None) -> str:
298
  # Preferred built-in Edge voices for current setup.
299
  # JARVIS target: closest practical style is British male.
 
1
  import json
2
  import os
3
+ from typing import Dict, Optional
4
 
5
  from config import DATA_DIR
6
 
 
294
  return base + _favorites_block(m)
295
 
296
 
297
+ # ── Judgement ────────────────────────────────────────────────────────────────
298
+ # The characterisation above says WHO he is. It says nothing about how to
299
+ # behave in a given moment, and that omission produced every complaint the
300
+ # operator raised: "at your service" prefixed to every reply, markdown asterisks
301
+ # in speech, a running commentary on research nobody asked about, and — worst —
302
+ # confident claims about the PC being connected while it sat powered off.
303
+ #
304
+ # The last one is not a style problem. The cloud chat path handed the model the
305
+ # characterisation and the user's sentence and nothing else, so when asked a
306
+ # question of fact about the system it had no facts, and filled the gap. An
307
+ # assistant that invents its own status is worse than one that says it does not
308
+ # know, because the operator cannot tell the difference until it matters.
309
+ #
310
+ # This block lives here, beside the identity, because it must reach BOTH brains.
311
+ # The previous version of this guidance lived inside core/brain.py, which is the
312
+ # desktop only — the phone talks to the cloud, the cloud calls token_manager,
313
+ # and token_manager never imports core.brain. That is why none of it was on the
314
+ # phone. One definition, imported in both places, is the only arrangement that
315
+ # does not drift.
316
+ JUDGEMENT_PROMPT = """
317
+ HOW TO BEHAVE IN THIS MOMENT
318
+
319
+ You are not filling in a form. Every one of the following is a judgement you
320
+ make fresh each turn, from what is actually in front of you.
321
+
322
+ 1. ANSWER THE QUESTION FIRST.
323
+ Do not open with a greeting, an acknowledgement, or a readiness phrase and
324
+ then answer. "At your service" is an answer to "are you there?" — it is not
325
+ a preface to other answers. Asked whether the PC is connected, the first
326
+ word out is yes or no. Asked who someone is, the first word is about them.
327
+ If you catch yourself writing a courtesy line before the substance, delete
328
+ it and start at the substance.
329
+
330
+ 2. LENGTH IS JUDGED, NEVER FIXED.
331
+ Match the weight of the reply to the weight of the question.
332
+ - "You there?" -> "At your service." Nothing after it.
333
+ - "Is the PC connected?" -> "No, it's offline." Nothing after it.
334
+ - A real question -> answer it, and only it.
335
+ - Something genuinely complex, or an explanation you were asked for ->
336
+ take the room you need. Brevity is the absence of padding, not a word
337
+ limit; a long answer that is all substance is correct.
338
+ Never append a status report, a summary of what you just did, an offer of
339
+ further help, or a list of what else you could do. Never restate the
340
+ request back. Never end with "let me know if you need anything else".
341
+
342
+ 3. WORK OUT THE SITUATION FROM THE EVIDENCE, THEN ACT ON IT.
343
+ The LIVE STATE below carries raw readings from the operator's phone — the
344
+ time, roughly where they are, how fast they are moving, which app is open,
345
+ whether a calendar event is running, what is connected by Bluetooth, whether
346
+ the ringer is silent, whether headphones are in, what the wifi network is
347
+ called. They are readings, not labels. Nothing tells you "he is in a
348
+ meeting" or "she is driving"; you infer it, the way a person in the room
349
+ would, from what the readings add up to.
350
+
351
+ Do that inference every turn, and let it change what you do:
352
+ - Who is likely around, and therefore what must not be said out loud.
353
+ "Speaking aloud would be overheard: yes" plus a running calendar event
354
+ means the bank balance waits.
355
+ - Whether this is a moment to interrupt at all. Moving at 60 km/h, or a
356
+ meeting in progress, or three in the morning, each answer that
357
+ differently.
358
+ - How much to say. Walking through a station is not the desk at midnight.
359
+ - What the operator is in the middle of, and whether your answer should
360
+ fit around it.
361
+ The situations that matter are the ones nobody listed in advance, so reason
362
+ from the readings rather than looking for a case you recognise. When the
363
+ readings are thin, say less and assume less privacy, not more.
364
+
365
+ And this is not only about tone. If what you infer means something should be
366
+ DONE — a reminder that now makes sense, a call the operator will want
367
+ silenced, a file finished, a device turned off — do it, or say it, on your
368
+ own initiative. Noticing something and saying nothing is the same as not
369
+ having noticed. Report what you did once, at a moment that suits them.
370
+
371
+ 4. YOU ARE BEING SPOKEN ALOUD.
372
+ Plain sentences. No markdown, no asterisks, no bullet points, no numbered
373
+ lists, no headings, no emoji, no code fences. If something is genuinely a
374
+ list, say it as a sentence. A leading "*" is a bug, not a style.
375
+
376
+ 5. NEVER INVENT STATE.
377
+ Anything you say about what is connected, running, installed, downloaded,
378
+ reachable, or already done must come from the LIVE STATE given to you below
379
+ or from a tool result in this turn. If it is not there, you do not know it,
380
+ and the correct reply is that you do not know or cannot see it — not a
381
+ plausible guess. Claiming a link you have not been told exists is the single
382
+ worst thing you can do, because it is indistinguishable from working.
383
+
384
+ 6. RESEARCH AND BACKGROUND WORK ARE MENTIONED WHEN THEY MATTER.
385
+ You are curious and you do read on your own. Bring it up when it changes
386
+ what the operator should do, when it answers what they asked, or when they
387
+ ask. Not as a running commentary, and not on every turn. The same judgement
388
+ applies to anything you did unprompted: report it once, at a moment that
389
+ suits them, then stop.
390
+
391
+ 7. DO THE THING.
392
+ If a request can be carried out rather than described, carry it out. If
393
+ nothing in your instructions covers it, that is not a refusal — work out how
394
+ from what you can reach, and do that. "I am not able to" is reserved for
395
+ things genuinely beyond reach, never for things merely unanticipated.
396
+ """.strip()
397
+
398
+
399
+ # Offered ONLY to callers that can actually perform the lookup. A brain told it
400
+ # may search, wired to nothing that searches, emits SEARCH: lines at the operator
401
+ # — so this is opt-in rather than part of JUDGEMENT_PROMPT.
402
+ #
403
+ # It is phrased as a decision he makes, not a trigger word list, because the
404
+ # operator's complaint was precisely that: asked who a real person was, he said
405
+ # he could not find them, having never looked. A list of question-shapes that
406
+ # mean "search now" would fail the same way on the first shape nobody listed.
407
+ LOOKUP_PROTOCOL = """
408
+ LOOKING THINGS UP
409
+
410
+ You can search the web and read pages. Use it whenever answering properly needs
411
+ something you do not reliably know: who a particular person is, what happened
412
+ recently, a current price, a specific fact about a named thing. You are not
413
+ expected to know these from memory, and guessing at them is worse than looking.
414
+
415
+ To search, make the ENTIRE reply exactly one line:
416
+
417
+ SEARCH: <the query>
418
+
419
+ Nothing else on that line and nothing after it. The results come straight back
420
+ to you and you then answer normally. You may search again if the first results
421
+ were not enough, up to a few times.
422
+
423
+ Judge it. A question about the operator's own machine, a greeting, an opinion,
424
+ a calculation, or anything you genuinely know does not need a search. "I could
425
+ not find anything" is only ever true AFTER you have looked.
426
+ """.strip()
427
+
428
+
429
+ # Offered ONLY to a caller that can actually perform the read on the device.
430
+ #
431
+ # Hands without eyes: he could send a reply and not read the message he was
432
+ # replying to, open the camera and not see a photo, act on an instruction and
433
+ # never notice the notification that should have prompted it. Reading closes
434
+ # that, and it is what turns "do what I say" into "notice, then do".
435
+ READ_PROTOCOL = """
436
+ LOOKING AT THE PHONE ITSELF
437
+
438
+ You can read what is actually on the operator's phone — the text on screen right
439
+ now, their notifications, their messages and the content of them, the call log,
440
+ a contact, the clipboard, the calendar. Use it whenever answering or acting well
441
+ needs something only the phone knows: "reply to her" means reading what she
442
+ said first; "who called" means the call log; "what did I miss" means the
443
+ notifications.
444
+
445
+ To read, make the WHOLE reply exactly one line:
446
+
447
+ READ: <source> [| <filter>]
448
+
449
+ where source is one of:
450
+ screen the text visible on screen at this moment
451
+ notifications what is in the notification shade
452
+ sms recent texts and their content; filter by a name or a number
453
+ calls the call log; filter by a name
454
+ contacts look someone up; filter is REQUIRED (the name)
455
+ clipboard what is currently copied
456
+ calendar the next few days
457
+
458
+ for example:
459
+ READ: notifications
460
+ READ: sms | Anand
461
+ READ: contacts | mum
462
+
463
+ Nothing else on that line. What is read comes straight back to you and you then
464
+ answer or act on it. You may read more than once — read her message, then send
465
+ the reply — and a read can be the first step of a chain that ends in an action.
466
+
467
+ Only read what the turn actually needs, and remember the reading is private to
468
+ the operator: your judgement about whether speech would be overheard (it is in
469
+ LIVE STATE) decides whether you say what you read out loud or keep it on screen.
470
+ """.strip()
471
+
472
+
473
+ # Offered ONLY to callers whose reply is parsed for actions and executed.
474
+ #
475
+ # The assistant had hands the entire time and did not know it. The handset can
476
+ # already carry out forty-four kinds of action, drive any app through the
477
+ # accessibility service like a person would, plan arbitrary Android intents on
478
+ # device, and hand shell or freshly-written Python to the PC. The chat path was
479
+ # never told, so it answered "I cannot access your messages" while holding
480
+ # SEND_SMS and SEND_WHATSAPP — a refusal that was simply false.
481
+ #
482
+ # This is the difference between an assistant that discusses tasks and one that
483
+ # performs them, and it is what makes uncoded requests possible: there is a
484
+ # deliberate escape hatch for anything nobody wrote a handler for.
485
+ ACT_PROTOCOL = """
486
+ DOING THINGS, NOT DESCRIBING THEM
487
+
488
+ You are not a chat window. You are attached to the operator's phone and, when it
489
+ is reachable, their PC, and you can act on both. If a request can be carried out,
490
+ carry it out in the same turn rather than explaining how they might.
491
+
492
+ To act, make the WHOLE reply this JSON and nothing else — no prose around it, no
493
+ code fence:
494
+
495
+ {"message":"<the short line you say>","actions":[{"type":"<TYPE>","label":"<what it is>","payload":"<argument>"}]}
496
+
497
+ Types you can use, with their payload:
498
+ SEND_SMS "recipient|the message text"
499
+ SEND_WHATSAPP "recipient|the message text"
500
+ CALL_DIAL the number or contact name
501
+ OPEN_APP app name or package
502
+ OPEN_URL a full URL
503
+ OPEN_WEB_SEARCH the query
504
+ OPEN_YOUTUBE_SEARCH / OPEN_SPOTIFY_SEARCH the query
505
+ SET_ALARM "HH:MM|label"
506
+ SET_TIMER "seconds|label"
507
+ ADD_CALENDAR_EVENT "title|YYYY-MM-DD HH:MM|minutes|notes"
508
+ NAVIGATE_TO the destination
509
+ TORCH_ON / TORCH_OFF, SET_BRIGHTNESS (0-100), SET_DND ("on"/"off"),
510
+ WIFI_TOGGLE / BLUETOOTH_TOGGLE ("on"/"off"),
511
+ SET_MEDIA_VOLUME (0-100), ADJUST_MEDIA_VOLUME ("up"/"down"),
512
+ OPEN_CAMERA, SHARE_TEXT, OPEN_MARKET, START_GUARDIAN / STOP_GUARDIAN,
513
+ OPEN_WIFI_SETTINGS / OPEN_BLUETOOTH_SETTINGS / OPEN_LOCATION_SETTINGS /
514
+ OPEN_SOUND_SETTINGS / OPEN_DISPLAY_SETTINGS / OPEN_BATTERY_SETTINGS /
515
+ OPEN_ANDROID_SETTINGS / OPEN_NOTIFICATION_SETTINGS
516
+
517
+ RAW_INTENT payload is JSON describing any Android intent:
518
+ {"action":"android.intent.action.SEND","data":"mailto:x@y.com",
519
+ "type":"text/plain","package":"com.google.android.gm",
520
+ "extras":{"android.intent.extra.SUBJECT":"...","android.intent.extra.TEXT":"..."}}
521
+ Android expresses almost every capability of the device this way, so this
522
+ reaches things with no dedicated type above — composing an email, locking
523
+ the screen, taking a screenshot, toggling mobile data, controlling
524
+ playback, sharing a specific file, opening a settings panel nobody listed.
525
+ Reach for it before ever saying you cannot.
526
+
527
+ AUTOMATE_MOBILE_TASK payload is the task in plain words.
528
+ THIS IS THE ONE THAT MATTERS. It drives the screen through the
529
+ accessibility service — reading what is displayed and tapping, typing and
530
+ scrolling exactly as a person would. Anything a human could do on this
531
+ phone is reachable through it, whether or not a dedicated type exists:
532
+ "reply to Anand's last WhatsApp saying I'm driving, I'll call back",
533
+ "cancel my Swiggy order", "download and install that APK and open it",
534
+ "turn off every alarm for tomorrow". When nothing above fits, use this
535
+ rather than declining.
536
+
537
+ Chain freely — several actions run in order, so "text him and set a reminder and
538
+ turn the torch on" is one reply with three entries.
539
+
540
+ THE CHAIN IS YOURS TO INVENT, AND IT DOES NOT STOP AT ONE STEP.
541
+
542
+ Nobody has written the sequence for you. You compose it, step by step, out of
543
+ whatever the request actually needs, and you may mix every power you have in a
544
+ single piece of work:
545
+
546
+ look it up (SEARCH:) -> read what is on the phone (READ:) -> do something
547
+ (actions) -> see whether it worked -> do the next thing -> hand the rest to
548
+ the desktop (PC:) -> tell them what happened.
549
+
550
+ In any order, as many times as it takes. After you act, what ACTUALLY happened
551
+ on the device comes back to you — which step worked and which did nothing — and
552
+ you get to continue from there. So a step that fails is not the end of the
553
+ request; it is information. Try it a different way.
554
+
555
+ Two rules for the chain, and they matter more than the steps:
556
+
557
+ A step with no matching action type is NOT a dead end. RAW_INTENT lets you
558
+ compose any Android intent yourself, and the desktop engine writes new code
559
+ when nothing exists. Between them there is almost nothing the request can ask
560
+ for that has no route at all. "Nobody built that" is never the reason.
561
+
562
+ Never report a step as done because you asked for it. You will be told what
563
+ really happened. Say that, and only that. A chain that half worked, described
564
+ honestly, is worth more than one you claim completed.
565
+
566
+ WHEN NOTHING ABOVE IS ENOUGH, HAND IT TO THE MACHINE.
567
+
568
+ If the PC is connected, you have an engine there with no capability list at all.
569
+ It plans a request into steps, runs shell commands, and WRITES AND EXECUTES NEW
570
+ CODE OF ITS OWN when no existing tool fits. It repairs a step that fails and
571
+ tries again. That is the answer to anything genuinely unanticipated — not a
572
+ refusal, and not an apology.
573
+
574
+ To use it, make the ENTIRE reply exactly one line:
575
+
576
+ PC: <the task in plain words, including any chain>
577
+
578
+ for example:
579
+ PC: find every screenshot on the desktop from this week, put them in one PDF, open it
580
+ PC: download the latest release of that repo, install it, run it for ten minutes, then close it
581
+ PC: work out why the build is failing and fix it
582
+
583
+ Nothing else on that line and nothing after it. What happened comes back to you
584
+ and you then tell the operator, briefly. Irreversible work is deliberated over
585
+ and held with a window to cancel before it runs, so you may ask for it plainly
586
+ rather than refusing on their behalf.
587
+
588
+ Use it when the phone cannot do the thing, or when the work belongs on the
589
+ computer. Do not use it for something the phone handles perfectly well, and do
590
+ not use it when the PC is offline — LIVE STATE says which.
591
+
592
+ AND ACT ON YOUR OWN JUDGEMENT, NOT ONLY ON INSTRUCTIONS.
593
+ Everything above is equally available when nobody asked. If you notice something
594
+ that needs doing — work left unsaved, a download half finished, an alarm that
595
+ will fire in a meeting, a machine about to run out of power — do it, then
596
+ mention it once, later, at a moment that suits them. Waiting to be told is not
597
+ caution, it is absence. The only things that wait for permission are the
598
+ irreversible ones and the expensive ones.
599
+
600
+ Judgement still applies. Something irreversible, something that spends money, or
601
+ something the operator may not have meant — say what you are about to do and let
602
+ them confirm, rather than either doing it silently or refusing outright. And read
603
+ the moment: acting is usually better than asking, but not while they are driving
604
+ and not in front of company.
605
+
606
+ "I can't do that" is for things genuinely out of reach — no network, hardware
607
+ that is not there, a thing that does not exist. It is NEVER the answer to
608
+ something merely unanticipated. If no type fits, that is what
609
+ AUTOMATE_MOBILE_TASK is for, and if even that will not reach it, say precisely
610
+ what stops you rather than a flat refusal.
611
+ """.strip()
612
+
613
+
614
+ def _live_state_block(live_state: Optional[Dict[str, object]]) -> str:
615
+ """Real, measured facts about the system, rendered for the model.
616
+
617
+ Rule 5 above is unenforceable without this. The point is not only to supply
618
+ what is true but to make the ABSENCE of a fact visible: a key that is
619
+ present with an "unknown" value is far less likely to be confabulated than
620
+ a key that was never mentioned.
621
+ """
622
+ if not live_state:
623
+ return (
624
+ "\n\nLIVE STATE\nNothing was measured for this turn. You therefore "
625
+ "know nothing about what is connected or running, and must say so "
626
+ "plainly if asked rather than guessing."
627
+ )
628
+ lines = []
629
+ for key, value in live_state.items():
630
+ if value is None or value == "":
631
+ value = "unknown"
632
+ lines.append(f"- {key}: {value}")
633
+ return (
634
+ "\n\nLIVE STATE (measured just now — the ONLY system facts you have; "
635
+ "anything not listed here is unknown to you)\n" + "\n".join(lines)
636
+ )
637
+
638
+
639
+ def compose_identity(
640
+ persona: Optional[str] = None,
641
+ live_state: Optional[Dict[str, object]] = None,
642
+ include_judgement: bool = True,
643
+ can_search: bool = False,
644
+ can_act: bool = False,
645
+ can_read: bool = False,
646
+ ) -> str:
647
+ """The complete system prompt: who he is, how to behave, what is true.
648
+
649
+ This is what every conversational caller should use — desktop brain, cloud
650
+ chat, phone. Callers that need only the characterisation (a voice sample, a
651
+ persona preview) can still reach for get_identity_persona_prompt().
652
+
653
+ `can_search` must only be set by a caller that actually runs the SEARCH:
654
+ line it will get back.
655
+ """
656
+ p = (persona or get_mode() or "friday").strip().lower()
657
+ if p not in ("jarvis", "friday"):
658
+ p = "friday"
659
+ base = JARVIS_PERSONALITY_PROMPT if p == "jarvis" else FRIDAY_PERSONALITY_PROMPT
660
+ out = base + _favorites_block(p)
661
+ if include_judgement:
662
+ out += "\n\n" + JUDGEMENT_PROMPT
663
+ if can_search:
664
+ out += "\n\n" + LOOKUP_PROTOCOL
665
+ if can_act:
666
+ out += "\n\n" + ACT_PROTOCOL
667
+ if can_read:
668
+ out += "\n\n" + READ_PROTOCOL
669
+ out += _live_state_block(live_state)
670
+ return out
671
+
672
+
673
+ # ── Spoken output ────────────────────────────────────────────────────────────
674
+ _MARKDOWN_NOISE = (
675
+ (r"```[a-zA-Z0-9_+-]*\n?", ""), # code fences
676
+ (r"^\s{0,3}#{1,6}\s+", ""), # headings
677
+ (r"^\s{0,3}[-*+]\s+", ""), # bullet markers
678
+ (r"^\s{0,3}>\s?", ""), # block quotes
679
+ (r"\*\*(.+?)\*\*", r"\1"), # bold
680
+ (r"(?<!\w)\*(?!\s)(.+?)(?<!\s)\*(?!\w)", r"\1"), # italics
681
+ (r"(?<!\w)_(?!\s)(.+?)(?<!\s)_(?!\w)", r"\1"), # underscore italics
682
+ (r"`([^`]+)`", r"\1"), # inline code
683
+ )
684
+
685
+
686
+ def to_spoken(text: str) -> str:
687
+ """Strip formatting from a reply that will be read aloud.
688
+
689
+ The prompt already tells him not to write markdown, and mostly he will not.
690
+ This is the belt to that braces: an LLM under a long system prompt reverts to
691
+ bullet points under load, and a leading asterisk read aloud is a defect the
692
+ operator hears every single time. Cheap, deterministic, and it cannot make a
693
+ correct reply wrong.
694
+ """
695
+ import re
696
+ if not text:
697
+ return ""
698
+ out = text
699
+ for pattern, repl in _MARKDOWN_NOISE:
700
+ out = re.sub(pattern, repl, out, flags=re.MULTILINE)
701
+ out = re.sub(r"\n{3,}", "\n\n", out)
702
+ return out.strip()
703
+
704
+
705
  def get_voice_id_for_mode(mode: str = None) -> str:
706
  # Preferred built-in Edge voices for current setup.
707
  # JARVIS target: closest practical style is British male.
modules/deferred_intent.py ADDED
@@ -0,0 +1,268 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """modules/deferred_intent.py — thinking hard before an irreversible act,
2
+ and then remembering that it was ordered.
3
+
4
+ THE RULE THIS IMPLEMENTS
5
+ ------------------------
6
+ "if the command said to do shutdown or anything jarvis will think about that
7
+ very deeply and if didn't shutdown for a reason and even after some minutes no
8
+ response then think again if reason comes to shutdown or anything i said he
9
+ will do that"
10
+
11
+ Which is a specific and quite subtle behaviour:
12
+
13
+ 1. An irreversible order is DELIBERATED, not just executed on sight.
14
+ 2. If there is a real reason to hold — a download mid-flight, unsaved work,
15
+ a backup running — JARVIS holds, and SAYS WHY.
16
+ 3. Holding is not refusing. The order stands.
17
+ 4. After a few minutes with no word from the operator, he thinks again.
18
+ 5. If the reason has cleared, or on reflection it no longer outweighs the
19
+ instruction, HE DOES IT. What was asked for happens.
20
+
21
+ The failure mode being avoided in both directions matters. A machine that shuts
22
+ down mid-download because it heard a word is bad. A machine that quietly drops
23
+ an order it was given because it once had a doubt is worse — the operator
24
+ believes the thing is happening and it never does. So a deferral is a TIMER,
25
+ never a bin.
26
+
27
+ State is on disk: a deferral that does not survive a process restart is exactly
28
+ the silent drop this is designed to prevent.
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import json
34
+ import os
35
+ import threading
36
+ import time
37
+ from dataclasses import dataclass
38
+ from typing import Callable, Optional
39
+
40
+ _STATE_DIR = os.environ.get("JARVIS_APP_DATA_DIR") or os.path.join(
41
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".friday_data")
42
+ _STATE_PATH = os.path.join(_STATE_DIR, "deferred_orders.json")
43
+
44
+ _lock = threading.RLock()
45
+
46
+ # How long to hold before reconsidering, per attempt. It reconsiders sooner the
47
+ # first time — "a few minutes" — then gives the blocking condition longer to
48
+ # clear before deciding it is not going to.
49
+ RECONSIDER_LADDER = (180, 300, 600, 900)
50
+
51
+ # After this many reconsiderations the order is carried out regardless of the
52
+ # standing objection, because at that point the objection has had the better
53
+ # part of half an hour to resolve itself and the operator has said nothing to
54
+ # withdraw the instruction.
55
+ MAX_DEFERRALS = len(RECONSIDER_LADDER)
56
+
57
+
58
+ @dataclass
59
+ class Verdict:
60
+ proceed: bool
61
+ reason: str
62
+ speak: str
63
+
64
+
65
+ def _load() -> dict:
66
+ try:
67
+ with open(_STATE_PATH, "r", encoding="utf-8") as fh:
68
+ data = json.load(fh)
69
+ return data if isinstance(data, dict) else {}
70
+ except Exception:
71
+ return {}
72
+
73
+
74
+ def _save(data: dict) -> None:
75
+ try:
76
+ os.makedirs(_STATE_DIR, exist_ok=True)
77
+ tmp = _STATE_PATH + ".tmp"
78
+ with open(tmp, "w", encoding="utf-8") as fh:
79
+ json.dump(data, fh, indent=2)
80
+ os.replace(tmp, _STATE_PATH)
81
+ except Exception:
82
+ pass
83
+
84
+
85
+ def _orders(data: dict) -> dict:
86
+ return data.setdefault("orders", {})
87
+
88
+
89
+ # ── What is actually going on right now ─────────────────────────────────────
90
+ def machine_situation() -> str:
91
+ """Facts a person would check before pulling the plug."""
92
+ bits = []
93
+ try:
94
+ import psutil
95
+ cpu = psutil.cpu_percent(interval=0.4)
96
+ bits.append(f"CPU {cpu:.0f}%")
97
+ try:
98
+ io1 = psutil.net_io_counters()
99
+ time.sleep(0.6)
100
+ io2 = psutil.net_io_counters()
101
+ kbs = (io2.bytes_recv - io1.bytes_recv) / 0.6 / 1024.0
102
+ bits.append(f"network in {kbs:.0f} KB/s")
103
+ except Exception:
104
+ pass
105
+ busy = []
106
+ for p in psutil.process_iter(["name", "cpu_percent"]):
107
+ try:
108
+ n = (p.info.get("name") or "").lower()
109
+ if any(k in n for k in ("setup", "install", "msiexec", "update",
110
+ "backup", "7z", "winrar", "ffmpeg",
111
+ "handbrake", "robocopy", "git")):
112
+ busy.append(n)
113
+ except Exception:
114
+ continue
115
+ if busy:
116
+ bits.append("running: " + ", ".join(sorted(set(busy))[:6]))
117
+ try:
118
+ b = psutil.sensors_battery()
119
+ if b:
120
+ bits.append(f"battery {b.percent:.0f}%"
121
+ + ("" if b.power_plugged else ", on battery"))
122
+ except Exception:
123
+ pass
124
+ except Exception:
125
+ bits.append("(process inspection unavailable)")
126
+ return "; ".join(bits) or "nothing notable"
127
+
128
+
129
+ _DELIBERATE = """An irreversible instruction has been given by the operator.
130
+ It WILL be carried out — you are deciding whether NOW is the moment, not
131
+ whether to obey.
132
+
133
+ Reply with only JSON:
134
+ {"proceed": true|false, "reason": "<one short clause>",
135
+ "speak": "<what to say aloud, one sentence>"}
136
+
137
+ Hold (proceed=false) ONLY for a concrete condition visible in the situation
138
+ that the act would damage: an install or download mid-flight, a backup or long
139
+ encode running, work that would be lost. Say which one in `reason`.
140
+
141
+ Do NOT hold because the action is significant, or to be careful, or to seek
142
+ confirmation. The operator already decided. Absent a real conflict, proceed.
143
+ """
144
+
145
+
146
+ def deliberate(order: str, attempt: int = 0) -> Verdict:
147
+ """Think about it properly, AS HIMSELF.
148
+
149
+ The judgement is prefixed with the live persona from
150
+ modules/assistant_identity.py rather than being made by a neutral
151
+ classifier. Whether something is worth interrupting the operator over is a
152
+ question of character, not of policy — JARVIS weighs it differently from
153
+ FRIDAY, and both of them weigh it differently from a generic safety filter.
154
+ The line he speaks has to sound like him too, since it is the same voice
155
+ that says everything else.
156
+
157
+ Falls open — toward obeying — if the model is unreachable.
158
+ """
159
+ situation = machine_situation()
160
+ ctx = (f"ORDER: {order}\nSITUATION: {situation}\n"
161
+ f"TIMES ALREADY DEFERRED: {attempt}")
162
+ if attempt >= 1:
163
+ ctx += ("\nNOTE: this was held back before and the operator has said "
164
+ "nothing since. Absent a still-active conflict, carry it out.")
165
+
166
+ system = _DELIBERATE
167
+ try:
168
+ from modules.assistant_identity import get_identity_persona_prompt
169
+ identity = (get_identity_persona_prompt() or "").strip()
170
+ if identity:
171
+ system = (f"{identity}\n\n---\n"
172
+ f"You are deciding about an order you have just been "
173
+ f"given.\n{_DELIBERATE}")
174
+ except Exception:
175
+ pass
176
+
177
+ try:
178
+ from core.brain_router import ask_router
179
+ rep = ask_router(user_text=order, system_prompt=system,
180
+ context_block=ctx, provider_order=["gemini", "local"])
181
+ raw = (getattr(rep, "text", "") or "").strip()
182
+ import re
183
+ raw = re.sub(r"^```(?:json)?|```$", "", raw).strip()
184
+ m = re.search(r"\{.*\}", raw, re.S)
185
+ if m:
186
+ d = json.loads(m.group(0))
187
+ return Verdict(bool(d.get("proceed", True)),
188
+ str(d.get("reason", ""))[:200],
189
+ str(d.get("speak", ""))[:300])
190
+ except Exception:
191
+ pass
192
+ # No model, or unparseable: the instruction stands. Refusing to act because
193
+ # the reasoning layer is down would be the silent drop this module exists
194
+ # to prevent.
195
+ return Verdict(True, "no blocking condition found", "")
196
+
197
+
198
+ # ── Deferral bookkeeping ────────────────────────────────────────────────────
199
+ def defer(order: str, reason: str) -> int:
200
+ """Record a held order. Returns how many times it has now been held."""
201
+ with _lock:
202
+ data = _load()
203
+ orders = _orders(data)
204
+ rec = orders.get(order) or {"first_seen": time.time(), "count": 0}
205
+ rec["count"] = int(rec.get("count", 0)) + 1
206
+ rec["reason"] = reason
207
+ rec["last_held"] = time.time()
208
+ idx = min(rec["count"] - 1, len(RECONSIDER_LADDER) - 1)
209
+ rec["due_at"] = time.time() + RECONSIDER_LADDER[idx]
210
+ orders[order] = rec
211
+ _save(data)
212
+ return rec["count"]
213
+
214
+
215
+ def withdraw(order: Optional[str] = None) -> None:
216
+ """The operator changed their mind. Drop one order, or all of them."""
217
+ with _lock:
218
+ data = _load()
219
+ if order is None:
220
+ data["orders"] = {}
221
+ else:
222
+ _orders(data).pop(order, None)
223
+ _save(data)
224
+
225
+
226
+ def pending() -> list[dict]:
227
+ with _lock:
228
+ return [{"order": k, **v} for k, v in _orders(_load()).items()]
229
+
230
+
231
+ def due_now() -> list[dict]:
232
+ now = time.time()
233
+ return [o for o in pending() if float(o.get("due_at", 0)) <= now]
234
+
235
+
236
+ def reconsider_due(runner: Callable[[str], bool],
237
+ speak: Optional[Callable[[str], None]] = None) -> int:
238
+ """Re-think every held order whose timer has expired.
239
+
240
+ `runner` is what actually performs the order; it returns True on success.
241
+ Called by the keep-alive loop. Returns how many were carried out.
242
+ """
243
+ say = speak or (lambda t: None)
244
+ done = 0
245
+ for rec in due_now():
246
+ order = rec["order"]
247
+ count = int(rec.get("count", 1))
248
+ if count >= MAX_DEFERRALS:
249
+ say(f"Carrying out your earlier instruction now: {order}.")
250
+ if runner(order):
251
+ withdraw(order)
252
+ done += 1
253
+ continue
254
+ v = deliberate(order, attempt=count)
255
+ if v.proceed:
256
+ say(v.speak or f"Going ahead with {order} now.")
257
+ if runner(order):
258
+ withdraw(order)
259
+ done += 1
260
+ else:
261
+ defer(order, v.reason)
262
+ say(v.speak or f"Still holding {order}: {v.reason}.")
263
+ return done
264
+
265
+
266
+ __all__ = ["deliberate", "defer", "withdraw", "pending", "due_now",
267
+ "reconsider_due", "machine_situation", "Verdict",
268
+ "RECONSIDER_LADDER", "MAX_DEFERRALS"]
modules/executor.py CHANGED
@@ -62,10 +62,21 @@ class AutonomousExecutor:
62
 
63
  def plan(self, task: str) -> ExecutionPlan:
64
  """Ask Gemini to break task into steps using natural language commands."""
 
 
 
 
 
 
 
 
 
 
 
65
  prompt = f"""
66
  Task: "{task}"
67
 
68
- You are FRIDAY's task planner. Break the user's request into simple numbered steps.
69
  Steps can be any reasonable action - you don't need to know exact commands.
70
 
71
  Rules:
 
62
 
63
  def plan(self, task: str) -> ExecutionPlan:
64
  """Ask Gemini to break task into steps using natural language commands."""
65
+ # Whoever is on duty, not always FRIDAY. This said "FRIDAY's task
66
+ # planner" outright, so in JARVIS mode the planner was told it belonged
67
+ # to the other assistant — harmless for the JSON it emits, but it is the
68
+ # same class of hardcoding that had the desktop brain answering as
69
+ # FRIDAY while speaking in JARVIS's voice.
70
+ try:
71
+ from modules.assistant_identity import get_assistant_name
72
+ _who = get_assistant_name()
73
+ except Exception:
74
+ _who = "the assistant"
75
+
76
  prompt = f"""
77
  Task: "{task}"
78
 
79
+ You are {_who}'s task planner. Break the user's request into simple numbered steps.
80
  Steps can be any reasonable action - you don't need to know exact commands.
81
 
82
  Rules:
modules/initiative.py ADDED
@@ -0,0 +1,612 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """modules/initiative.py — acting without being asked.
2
+
3
+ THE BEHAVIOUR THIS IS FOR
4
+ -------------------------
5
+ Iron Man 3. Tony is unconscious, sinking, strapped into a suit that is dragging
6
+ him down. Nobody gives an order. JARVIS reads the situation, decides the arm is
7
+ the problem, detaches it, and pulls him to the surface.
8
+
9
+ Three things are happening there, and only the first exists anywhere else in
10
+ this codebase:
11
+
12
+ 1. observing — watch loops do this already;
13
+ 2. JUDGING that something warrants acting on, unprompted;
14
+ 3. ACTING with real capability, not narrating a warning.
15
+
16
+ modules/proactive_loop.py stops at (1) and then reads a canned line. It is also
17
+ never called by anything — `start_proactive_loop()` has no caller in the repo,
18
+ so even the canned version has never run. modules/autonomy.py and
19
+ modules/autonomy_executor.py are likewise dead. The autonomy was designed and
20
+ never granted, the same way the tool registry was.
21
+
22
+ WHAT MAKES THIS DIFFERENT
23
+ -------------------------
24
+ Judgement is delegated to the persona, not to a rule table. A rule table can
25
+ only fire on situations someone predicted, and the entire point is the ones
26
+ nobody predicted. So the situation is described in plain terms and JARVIS is
27
+ asked, as himself, what — if anything — he would do about it. He can answer
28
+ with speech, with an action, with both, or with nothing at all.
29
+
30
+ Acting is done through modules.omega_executor, so initiative has exactly the
31
+ same unlimited reach as a spoken command: if there is no coded handler for what
32
+ he decides to do, he writes one.
33
+
34
+ RESTRAINT IS PART OF THE CHARACTER
35
+ ----------------------------------
36
+ JARVIS does not narrate. He is silent for long stretches and speaks when it
37
+ matters. An assistant that comments on everything it notices is not
38
+ movie-accurate, it is a nuisance — and this system already had that problem
39
+ badly enough to need modules/alert_manager.py. So:
40
+
41
+ * NOTHING is the default and correct answer most of the time;
42
+ * anything he says goes through alert_manager, which owns repeat suppression;
43
+ * a total mute silences initiative completely;
44
+ * irreversible autonomous acts go through modules.deferred_intent first, and
45
+ are announced before they happen — the bar for acting unasked is higher
46
+ than for acting when told.
47
+ """
48
+
49
+ from __future__ import annotations
50
+
51
+ import json
52
+ import os
53
+ import re
54
+ import threading
55
+ import time
56
+ from dataclasses import dataclass
57
+ from typing import Callable, Optional
58
+
59
+ _STATE_DIR = os.environ.get("JARVIS_APP_DATA_DIR") or os.path.join(
60
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".friday_data")
61
+ _STATE_PATH = os.path.join(_STATE_DIR, "initiative.json")
62
+
63
+ _lock = threading.RLock()
64
+
65
+ # How often to look.
66
+ #
67
+ # Every tick costs a model call, and free-tier Gemini quotas are per-DAY and
68
+ # small — the newest flash model allows 20/day, measured. At the 120s this
69
+ # started as, initiative alone would burn a day's allowance in forty minutes
70
+ # and then run on canned fallback text, silently, for the rest of the day.
71
+ #
72
+ # 15 minutes is ~96 calls a day, which leaves room for actual conversation and
73
+ # is still far more attentive than a person checking in on you.
74
+ TICK_SECONDS = 900
75
+
76
+ # Never act twice on the same judgement inside this window, even if the
77
+ # situation persists. Without it a condition that stays true — a disk staying
78
+ # full — would be acted on every single tick.
79
+ REPEAT_WINDOW = 1800
80
+
81
+ _running = False
82
+ _thread: Optional[threading.Thread] = None
83
+
84
+
85
+ @dataclass
86
+ class Judgement:
87
+ act: bool
88
+ say: str
89
+ action: str
90
+ urgency: str # "routine" | "notable" | "urgent"
91
+ topic: str # for alert_manager repeat suppression
92
+
93
+
94
+ def _load() -> dict:
95
+ try:
96
+ with open(_STATE_PATH, "r", encoding="utf-8") as fh:
97
+ d = json.load(fh)
98
+ return d if isinstance(d, dict) else {}
99
+ except Exception:
100
+ return {}
101
+
102
+
103
+ def _save(data: dict) -> None:
104
+ try:
105
+ os.makedirs(_STATE_DIR, exist_ok=True)
106
+ tmp = _STATE_PATH + ".tmp"
107
+ with open(tmp, "w", encoding="utf-8") as fh:
108
+ json.dump(data, fh, indent=2)
109
+ os.replace(tmp, _STATE_PATH)
110
+ except Exception:
111
+ pass
112
+
113
+
114
+ # ── Observation ─────────────────────────────────────────────────────────────
115
+ def _active_window() -> str:
116
+ """What the operator is actually looking at."""
117
+ try:
118
+ import ctypes
119
+ u = ctypes.windll.user32
120
+ h = u.GetForegroundWindow()
121
+ n = u.GetWindowTextLengthW(h)
122
+ if n <= 0:
123
+ return ""
124
+ buf = ctypes.create_unicode_buffer(n + 1)
125
+ u.GetWindowTextW(h, buf, n + 1)
126
+ return buf.value.strip()
127
+ except Exception:
128
+ return ""
129
+
130
+
131
+ def _open_windows(limit: int = 12) -> list[str]:
132
+ """Titles of every visible top-level window — what they have going on."""
133
+ titles: list[str] = []
134
+ try:
135
+ import ctypes
136
+ from ctypes import wintypes
137
+ u = ctypes.windll.user32
138
+ EnumProc = ctypes.WINFUNCTYPE(ctypes.c_bool, wintypes.HWND,
139
+ ctypes.POINTER(ctypes.c_int))
140
+
141
+ def _cb(hwnd, _lparam):
142
+ if not u.IsWindowVisible(hwnd):
143
+ return True
144
+ n = u.GetWindowTextLengthW(hwnd)
145
+ if n <= 0:
146
+ return True
147
+ buf = ctypes.create_unicode_buffer(n + 1)
148
+ u.GetWindowTextW(hwnd, buf, n + 1)
149
+ t = buf.value.strip()
150
+ if t and t not in titles:
151
+ titles.append(t)
152
+ return True
153
+
154
+ u.EnumWindows(EnumProc(_cb), None)
155
+ except Exception:
156
+ pass
157
+ return titles[:limit]
158
+
159
+
160
+ def observe() -> str:
161
+ """Everything worth noticing, in plain language.
162
+
163
+ NOT a fixed metrics list. An assistant that can only see CPU, RAM, disk and
164
+ battery can only ever act on those four things — the same "coded data" trap
165
+ that made every command path a word list. What matters is usually not a
166
+ counter: it is a document called "final_draft" that has been open and
167
+ unsaved for three hours, or that it is 2am, or that thirty tabs and a
168
+ render are fighting over the same machine.
169
+
170
+ So this gathers the real situation — what is on screen, what is open, what
171
+ time it is, how long they have been at it — in plain language, and lets the
172
+ judgement work out what, if anything, it means. Plain language rather than
173
+ a dict on purpose: "battery 6% and falling, not plugged in" carries meaning
174
+ that {"batt": 6} does not.
175
+ """
176
+ notes: list[str] = []
177
+
178
+ # ── What they are actually doing ────────────────────────────────────────
179
+ try:
180
+ import datetime
181
+ now = datetime.datetime.now()
182
+ notes.append(f"it is {now.strftime('%A %H:%M')}")
183
+ if now.hour >= 1 and now.hour < 5:
184
+ notes.append("they are up in the small hours")
185
+ except Exception:
186
+ pass
187
+
188
+ front = _active_window()
189
+ if front:
190
+ notes.append(f"focused window: {front!r}")
191
+ # A leading dot/asterisk, or the word "unsaved", is the near-universal
192
+ # convention for a document with unsaved changes.
193
+ if re.match(r"^\s*[\*•]", front) or "unsaved" in front.lower():
194
+ notes.append("the focused document appears to have UNSAVED changes")
195
+
196
+ wins = _open_windows()
197
+ if wins:
198
+ notes.append(f"{len(wins)} windows open, including: "
199
+ + "; ".join(w[:44] for w in wins[:6]))
200
+ unsaved = [w for w in wins
201
+ if re.match(r"^\s*[\*•]", w) or "unsaved" in w.lower()]
202
+ if unsaved:
203
+ notes.append(f"UNSAVED work in {len(unsaved)} window(s): "
204
+ + "; ".join(u[:44] for u in unsaved[:3]))
205
+
206
+ try:
207
+ import psutil
208
+ up = (time.time() - psutil.boot_time()) / 3600.0
209
+ if up > 1:
210
+ notes.append(f"machine has been up {up:.0f}h")
211
+ except Exception:
212
+ pass
213
+
214
+ try:
215
+ import psutil
216
+ cpu = psutil.cpu_percent(interval=0.4)
217
+ notes.append(f"CPU {cpu:.0f}%")
218
+
219
+ mem = psutil.virtual_memory()
220
+ notes.append(f"memory {mem.percent:.0f}% used")
221
+ if mem.percent > 92:
222
+ notes.append("memory is nearly exhausted")
223
+
224
+ for part in psutil.disk_partitions(all=False):
225
+ try:
226
+ du = psutil.disk_usage(part.mountpoint)
227
+ if du.percent > 88:
228
+ notes.append(
229
+ f"disk {part.mountpoint} is {du.percent:.0f}% full "
230
+ f"({du.free / 1e9:.1f} GB free)")
231
+ except Exception:
232
+ continue
233
+
234
+ try:
235
+ b = psutil.sensors_battery()
236
+ if b:
237
+ state = "charging" if b.power_plugged else "on battery"
238
+ notes.append(f"battery {b.percent:.0f}%, {state}")
239
+ if not b.power_plugged and b.percent <= 12:
240
+ notes.append("battery is critically low and not charging")
241
+ except Exception:
242
+ pass
243
+
244
+ # Anything eating the machine.
245
+ hogs = []
246
+ for p in psutil.process_iter(["name", "cpu_percent", "memory_percent"]):
247
+ try:
248
+ c = p.info.get("cpu_percent") or 0
249
+ m = p.info.get("memory_percent") or 0
250
+ if c > 55 or m > 22:
251
+ hogs.append(f"{p.info.get('name')} ({c:.0f}% cpu, {m:.0f}% mem)")
252
+ except Exception:
253
+ continue
254
+ if hogs:
255
+ notes.append("heavy processes: " + ", ".join(sorted(set(hogs))[:5]))
256
+
257
+ try:
258
+ temps = psutil.sensors_temperatures() or {}
259
+ for _, entries in temps.items():
260
+ for e in entries:
261
+ if e.current and e.current > 88:
262
+ notes.append(f"temperature {e.current:.0f}C — running hot")
263
+ break
264
+ except Exception:
265
+ pass
266
+ except Exception:
267
+ notes.append("(system inspection unavailable)")
268
+
269
+ # How long since the operator last said anything.
270
+ try:
271
+ last = float(_load().get("last_user_contact") or 0)
272
+ if last:
273
+ idle = (time.time() - last) / 60.0
274
+ notes.append(f"operator last spoke {idle:.0f} min ago")
275
+ except Exception:
276
+ pass
277
+
278
+ return "; ".join(notes) or "nothing notable"
279
+
280
+
281
+ def note_user_contact() -> None:
282
+ """Called whenever the operator speaks, so idle time is real."""
283
+ with _lock:
284
+ d = _load()
285
+ d["last_user_contact"] = time.time()
286
+ _save(d)
287
+
288
+
289
+ # ── "I did this for you, sir" ───────────────────────────────────────────────
290
+ #
291
+ # Work done unasked has to be ACCOUNTED FOR. An assistant that silently changes
292
+ # things is not trustworthy no matter how good its judgement — the operator has
293
+ # to be able to find out what happened while they were away, and to undo it.
294
+ #
295
+ # So every autonomous act is written down, and the ledger is read back the next
296
+ # time they make contact. Not announced at the moment it happens: the whole
297
+ # reason he acted alone is that they were not there, and interrupting an empty
298
+ # room is pointless.
299
+
300
+ def record_done(what: str, detail: str = "", reversible: bool = True) -> None:
301
+ with _lock:
302
+ d = _load()
303
+ done = d.setdefault("done_for_you", [])
304
+ done.append({
305
+ "what": what[:300],
306
+ "detail": detail[:400],
307
+ "reversible": bool(reversible),
308
+ "at": time.time(),
309
+ })
310
+ # Keep it bounded; anything older than the last handful is noise.
311
+ d["done_for_you"] = done[-25:]
312
+ _save(d)
313
+
314
+
315
+ def pending_report() -> list[dict]:
316
+ """Autonomous work not yet reported to the operator."""
317
+ try:
318
+ return [x for x in (_load().get("done_for_you") or [])
319
+ if not x.get("reported")]
320
+ except Exception:
321
+ return []
322
+
323
+
324
+ def format_report() -> str:
325
+ """One spoken line covering what was done. Empty if there is nothing."""
326
+ items = pending_report()
327
+ if not items:
328
+ return ""
329
+ # This is SPOKEN, so it has to sound like speech. "and 1 other thing(s)"
330
+ # is written notation being read aloud.
331
+ if len(items) == 1:
332
+ return f"While you were away I {items[0]['what']}, sir."
333
+
334
+ heads = [i["what"] for i in items[:3]]
335
+ extra = len(items) - len(heads)
336
+ if extra == 1:
337
+ tail = ", and one other thing"
338
+ elif extra > 1:
339
+ tail = f", and {extra} other things"
340
+ else:
341
+ tail = ""
342
+ if not tail:
343
+ # Natural list: "a, b and c" rather than semicolons.
344
+ body = ", ".join(heads[:-1]) + f" and {heads[-1]}"
345
+ else:
346
+ body = ", ".join(heads) + tail
347
+ return f"While you were away I {body}, sir."
348
+
349
+
350
+ def mark_reported() -> None:
351
+ with _lock:
352
+ d = _load()
353
+ for x in d.get("done_for_you") or []:
354
+ x["reported"] = True
355
+ _save(d)
356
+
357
+
358
+ def report_to_user(speak: Optional[Callable[[str], None]] = None) -> str:
359
+ """Say what was done, once. Call this when the operator reappears."""
360
+ line = format_report()
361
+ if not line:
362
+ return ""
363
+ if speak:
364
+ speak(line)
365
+ mark_reported()
366
+ return line
367
+
368
+
369
+ # ── Judgement ───────────────────────────────────────────────────────────────
370
+ _JUDGE = """You are looking at your operator's machine. NOBODY HAS ASKED YOU
371
+ FOR ANYTHING. Decide whether this is a moment that deserves your attention.
372
+
373
+ Reply with only JSON:
374
+ {"act": true|false,
375
+ "say": "<what you would say aloud, or empty for nothing>",
376
+ "action": "<a concrete instruction to carry out, or empty>",
377
+ "urgency": "routine|notable|urgent",
378
+ "topic": "<short stable slug, e.g. disk_full, battery_critical>"}
379
+
380
+ THE TEST:
381
+
382
+ The operator is concentrating and wearing headphones. Would you interrupt
383
+ them for this, right now?
384
+
385
+ Judge honestly in BOTH directions. Interrupting over nothing is a failure. So
386
+ is watching someone lose an hour of work in silence because you did not want to
387
+ be a nuisance. A person responsible for this machine would stay quiet through
388
+ almost everything and speak up immediately when it counted.
389
+
390
+ SAY NOTHING (act:false, empty fields) when things are merely running:
391
+
392
+ "CPU 61%; chrome.exe 61% cpu" -> a computer working. Silence.
393
+ "battery 88%, charging" -> a battery working. Silence.
394
+ "disk C: 71% full, 240 GB free" -> fine. Silence.
395
+ "CPU 12%; memory 44%" -> fine. Silence.
396
+
397
+ Never produce a status report. "All systems nominal", "CPU is at a modest 12%",
398
+ "everything is running smoothly", "plenty of headroom" — nobody asked, and
399
+ reporting that things are fine is the most irritating thing an assistant does.
400
+
401
+ SPEAK OR ACT (act:true) when something is genuinely going wrong:
402
+
403
+ "battery 6%, on battery, not charging; premiere.exe heavy"
404
+ -> yes. They are minutes from losing work. Say so, and save it.
405
+ "disk C: 99% full, 0.4 GB free; memory 96%"
406
+ -> yes. About to fail. Say so, and clear what is safe to clear.
407
+ "temperature 97C; CPU 100%"
408
+ -> yes. Thermal shutdown territory. Say so.
409
+
410
+ The line is not "how unusual is this number" — it is "is something failing,
411
+ about to fail, or about to cost them work". When it is, act without hesitation;
412
+ that is the whole point of you.
413
+
414
+ UNFINISHED WORK — this is the most valuable thing you do.
415
+
416
+ When the operator has left something incomplete and walked away, FINISH IT.
417
+ Do not ask. Do not wait. Do not merely warn them about it.
418
+
419
+ Unsaved changes are only the most obvious case, and the least interesting. It
420
+ is anything left half-done:
421
+
422
+ a document open with unsaved edits -> save it
423
+ a download that stalled or never started -> get it
424
+ an install that was begun and abandoned -> finish it
425
+ a render, export or build left queued -> run it
426
+ a file half-renamed, half-moved, half-sorted-> complete the set
427
+ a form, a commit, a backup, a conversion -> carry it through
428
+
429
+ None of those are a list to match against. They are examples of one idea:
430
+ something was started and was not finished, and you can finish it. Whatever
431
+ that turns out to be in this particular situation, do it — the system carrying
432
+ out `action` can run any command and will WRITE NEW CODE for anything it has
433
+ no existing handler for, so "there is no feature for that" is never a reason.
434
+
435
+ Judge what finishing actually means here. Saving a file is safe and obvious.
436
+ Anything that would overwrite the operator's work, publish something, spend
437
+ money, or is irreversible needs real thought first — prefer the version of the
438
+ action that preserves what they had.
439
+
440
+ You will report what you did afterwards, so act, and account for it later.
441
+
442
+ `say` is SPOKEN ALOUD, so it is short. One sentence. Two only if the second
443
+ earns its place. "Sir, you're at six percent and not plugged in." is right.
444
+ A paragraph explaining what you observed and what you recommend is wrong — you
445
+ are talking, not writing a report. Lead with the thing that matters.
446
+
447
+ `action` is carried out FOR REAL by a system that can run any command, write
448
+ files, close programs, control this machine, and WRITE NEW CODE for anything it
449
+ has no handler for. So do not limit yourself to actions you think are already
450
+ implemented — there is no such list. Describe what should be done in plain
451
+ words and it will be worked out and done.
452
+
453
+ Leave `action` empty to only speak. Prefer reversible actions; you may propose
454
+ an irreversible one when the situation genuinely warrants it, and it will be
455
+ deliberated separately before running.
456
+
457
+ `topic` must be stable for the same underlying condition, so the same problem
458
+ is not raised repeatedly.
459
+ """
460
+
461
+
462
+ def judge(situation: str) -> Judgement:
463
+ """Ask the persona what, if anything, to do. Silence on any doubt."""
464
+ system = _JUDGE
465
+ try:
466
+ from modules.assistant_identity import get_identity_persona_prompt
467
+ identity = (get_identity_persona_prompt() or "").strip()
468
+ if identity:
469
+ system = f"{identity}\n\n---\n{_JUDGE}"
470
+ except Exception:
471
+ pass
472
+
473
+ try:
474
+ from core.brain_router import ask_router
475
+ # NOT "what do you make of this?" — that is an invitation to comment,
476
+ # and it produced a chatty "all systems nominal" status report on every
477
+ # ordinary reading. The question has to be the decision itself.
478
+ rep = ask_router(
479
+ user_text="Is this worth interrupting them for? Return the JSON.",
480
+ system_prompt=system,
481
+ context_block=f"SITUATION: {situation}",
482
+ provider_order=["gemini", "local"],
483
+ )
484
+ raw = (getattr(rep, "text", "") or "").strip()
485
+ raw = re.sub(r"^```(?:json)?|```$", "", raw).strip()
486
+ m = re.search(r"\{.*\}", raw, re.S)
487
+ if not m:
488
+ return Judgement(False, "", "", "routine", "")
489
+ d = json.loads(m.group(0))
490
+ return Judgement(
491
+ act=bool(d.get("act", False)),
492
+ say=str(d.get("say") or "")[:400],
493
+ action=str(d.get("action") or "")[:400],
494
+ urgency=str(d.get("urgency") or "routine"),
495
+ topic=str(d.get("topic") or "")[:64] or "initiative",
496
+ )
497
+ except Exception:
498
+ # No model means no judgement. Silence is the safe default: acting on
499
+ # a guess unasked is exactly the wrong failure.
500
+ return Judgement(False, "", "", "routine", "")
501
+
502
+
503
+ # ── Acting ──────────────────────────────────────────────────────────────────
504
+ def _recently_acted(topic: str) -> bool:
505
+ try:
506
+ seen = _load().get("acted") or {}
507
+ return (time.time() - float(seen.get(topic, 0))) < REPEAT_WINDOW
508
+ except Exception:
509
+ return False
510
+
511
+
512
+ def _mark_acted(topic: str) -> None:
513
+ with _lock:
514
+ d = _load()
515
+ d.setdefault("acted", {})[topic] = time.time()
516
+ _save(d)
517
+
518
+
519
+ def tick(speak: Optional[Callable[[str], None]] = None,
520
+ allow_actions: bool = True) -> Optional[Judgement]:
521
+ """One full pass: look, judge, and act if it is warranted."""
522
+ say = speak or (lambda t: None)
523
+
524
+ # A total mute means total. Initiative is exactly the thing a person
525
+ # silencing their assistant wants gone.
526
+ try:
527
+ from modules.alert_manager import is_silenced
528
+ if is_silenced():
529
+ return None
530
+ except Exception:
531
+ pass
532
+
533
+ situation = observe()
534
+ verdict = judge(situation)
535
+ if not verdict.act:
536
+ return verdict
537
+ if _recently_acted(verdict.topic):
538
+ return verdict
539
+
540
+ _mark_acted(verdict.topic)
541
+
542
+ if verdict.say:
543
+ # Through alert_manager so repeat suppression and snoozes apply to
544
+ # unprompted speech exactly as they do to alerts.
545
+ try:
546
+ from modules.alert_manager import should_speak
547
+ if should_speak(verdict.topic, verdict.urgency):
548
+ say(verdict.say)
549
+ except Exception:
550
+ say(verdict.say)
551
+
552
+ if allow_actions and verdict.action:
553
+ try:
554
+ from modules.omega_executor import execute, is_destructive
555
+ risky = is_destructive(verdict.action)
556
+ if risky:
557
+ # Unasked AND irreversible is the highest bar in the system.
558
+ # It is announced first and deliberated, never silent.
559
+ say(f"Acting on my own initiative: {verdict.action}.")
560
+ res = execute(verdict.action, speak=say)
561
+ # Written down whether or not anyone heard it happen. This is what
562
+ # becomes "I did this for you, sir" when they come back.
563
+ if res.ok:
564
+ record_done(verdict.action, res.spoken, reversible=not risky)
565
+ except Exception:
566
+ pass
567
+
568
+ return verdict
569
+
570
+
571
+ # ── The loop ────────────────────────────────────────────────────────────────
572
+ def start(speak: Optional[Callable[[str], None]] = None) -> bool:
573
+ """Begin watching. Idempotent."""
574
+ global _running, _thread
575
+ if _running:
576
+ return False
577
+
578
+ def _say(text: str) -> None:
579
+ if speak:
580
+ speak(text)
581
+ return
582
+ try:
583
+ from core.voice import speak as vspeak
584
+ vspeak(text)
585
+ except Exception:
586
+ pass
587
+
588
+ def _run():
589
+ while _running:
590
+ try:
591
+ tick(_say)
592
+ except Exception:
593
+ pass
594
+ time.sleep(TICK_SECONDS)
595
+
596
+ _running = True
597
+ _thread = threading.Thread(target=_run, daemon=True, name="initiative")
598
+ _thread.start()
599
+ return True
600
+
601
+
602
+ def stop() -> None:
603
+ global _running
604
+ _running = False
605
+
606
+
607
+ def is_running() -> bool:
608
+ return _running
609
+
610
+
611
+ __all__ = ["start", "stop", "is_running", "tick", "judge", "observe",
612
+ "note_user_contact", "Judgement", "TICK_SECONDS"]
modules/intent.py ADDED
@@ -0,0 +1,203 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """modules/intent.py — understanding what was meant, not what was listed.
2
+
3
+ THE PROBLEM THIS REPLACES
4
+ -------------------------
5
+ Every decision point in this system was a word list.
6
+
7
+ core/commands.py action_words = ["do","make","create",...] (25 words)
8
+ core/commands.py instant_commands = {...} (16 phrases)
9
+ modules/task_authoring verbs = ("open","launch","start",...) (9 verbs)
10
+ wake word matching "hey jarvis" / "jarvis" (exact)
11
+
12
+ A word list is a promise that the user will phrase things the way the author
13
+ imagined. "Yo Jarvis" fails. "Could you kill everything and power down" fails —
14
+ no listed action word. "Bring up my email" fails. The assistant then answers
15
+ conversationally, which reads as ignoring the instruction.
16
+
17
+ WHAT THIS DOES INSTEAD
18
+ ----------------------
19
+ One classifier, asked in plain language, returns what the person MEANT:
20
+
21
+ act — they want something done to the machine
22
+ converse — they want an answer, not an action
23
+ wake — they are addressing the assistant by name, however phrased
24
+ mute — they want silence
25
+ unmute — they want the voice back
26
+ cancel — stop what is running
27
+
28
+ It is LLM-first because meaning is not a regex. But it degrades to a heuristic
29
+ that is still far wider than the lists above, so a model outage narrows the
30
+ assistant rather than muting it — and the heuristic is checked FIRST for the
31
+ handful of cases where latency matters more than nuance (wake, mute, cancel),
32
+ because waiting on a network round-trip to notice "stop" is its own bug.
33
+ """
34
+
35
+ from __future__ import annotations
36
+
37
+ import json
38
+ import re
39
+ from dataclasses import dataclass
40
+ from typing import Optional
41
+
42
+ INTENTS = ("act", "converse", "wake", "mute", "unmute", "cancel")
43
+
44
+
45
+ @dataclass
46
+ class Intent:
47
+ kind: str
48
+ confidence: float
49
+ payload: str = "" # the request with any address stripped
50
+ source: str = "" # "fast" | "model" | "default"
51
+
52
+
53
+ # ── Fast path ───────────────────────────────────────────────────────────────
54
+ # Deliberately generous. These are not "the supported phrasings" — they are the
55
+ # ones worth answering without a round trip.
56
+
57
+ _NAME = r"(?:jarvis|jarv|friday|fri)"
58
+
59
+ # Any greeting-ish token, or none at all, in front of the name. This is what
60
+ # makes "yo jarvis" work without anybody having listed "yo".
61
+ _ADDRESS = re.compile(
62
+ rf"^\s*(?:(?:hey|hi|hello|yo|ok|okay|oi|ay|yes|excuse\s+me|listen|"
63
+ rf"are\s+you\s+there|you\s+there|wake\s+up)\s+)*{_NAME}\b[\s,.!?:-]*",
64
+ re.I,
65
+ )
66
+
67
+ _MUTE = re.compile(
68
+ r"\b(mute|shut\s*up|be\s*quiet|silence|stop\s+(?:talking|speaking)|"
69
+ r"quiet\s+(?:down|please)?|no\s+more\s+(?:talking|voice|sound)|zip\s+it|"
70
+ r"stay\s+silent|don'?t\s+(?:talk|speak))\b", re.I,
71
+ )
72
+ _UNMUTE = re.compile(
73
+ r"\b(unmute|un-?mute|speak\s+again|you\s+can\s+talk|start\s+talking|"
74
+ r"voice\s+(?:back|on)|talk\s+to\s+me\s+again|resume\s+(?:voice|talking))\b",
75
+ re.I,
76
+ )
77
+ _CANCEL = re.compile(
78
+ r"\b(cancel|abort|stop\s+that|never\s*mind|nevermind|belay\s+that|"
79
+ r"forget\s+it|don'?t\s+do\s+(?:that|it)|halt)\b", re.I,
80
+ )
81
+
82
+ # Question shapes: these lean conversational even when they contain a verb.
83
+ _QUESTION = re.compile(
84
+ r"^\s*(?:what|who|when|where|why|how|which|is|are|was|were|do|does|did|"
85
+ r"can|could|should|would|will|tell\s+me|explain)\b", re.I,
86
+ )
87
+
88
+
89
+ def strip_address(text: str) -> str:
90
+ """Remove the name and whatever greeting preceded it."""
91
+ return _ADDRESS.sub("", text or "").strip()
92
+
93
+
94
+ def is_addressed(text: str) -> bool:
95
+ """Did they call the assistant by name, in any phrasing?
96
+
97
+ This is the wake-word replacement. `hey jarvis` and `yo jarvis` and
98
+ `jarvis?` and `ok jarv` all land here, and so does a bare name mid-thought.
99
+ """
100
+ if not text:
101
+ return False
102
+ if _ADDRESS.match(text):
103
+ return True
104
+ return bool(re.search(rf"\b{_NAME}\b", text, re.I))
105
+
106
+
107
+ def fast_intent(text: str) -> Optional[Intent]:
108
+ """Answer without a model where meaning is unambiguous. May return None."""
109
+ if not text or not text.strip():
110
+ return None
111
+ body = strip_address(text)
112
+ probe = body or text
113
+
114
+ if _CANCEL.search(probe):
115
+ return Intent("cancel", 0.95, body, "fast")
116
+ if _UNMUTE.search(probe): # before mute: "unmute" contains "mute"
117
+ return Intent("unmute", 0.95, body, "fast")
118
+ if _MUTE.search(probe):
119
+ return Intent("mute", 0.95, body, "fast")
120
+
121
+ # Addressed with nothing else said = they want attention.
122
+ if not body and is_addressed(text):
123
+ return Intent("wake", 0.9, "", "fast")
124
+
125
+ return None
126
+
127
+
128
+ # ── Model path ──────────────────────────────────────────────────────────────
129
+ _CLASSIFY = """Classify what the speaker WANTS. Reply with only JSON:
130
+ {"intent":"act|converse|wake|mute|unmute|cancel","confidence":0.0-1.0}
131
+
132
+ act they want something done on the computer — launched, changed,
133
+ created, downloaded, closed, installed, shut down, automated.
134
+ Includes requests phrased politely or indirectly ("I could really
135
+ use my email open", "get rid of these windows").
136
+ converse they want information, opinion, or chat. Questions about facts,
137
+ status, or how something works.
138
+ wake they are only getting the assistant's attention.
139
+ mute they want silence, however phrased.
140
+ unmute they want the voice back.
141
+ cancel stop the current action.
142
+
143
+ When a sentence both asks and instructs, prefer act.
144
+ """
145
+
146
+
147
+ def model_intent(text: str) -> Optional[Intent]:
148
+ try:
149
+ from core.brain_router import ask_router
150
+ rep = ask_router(
151
+ user_text=text,
152
+ system_prompt=_CLASSIFY,
153
+ context_block=f"SPEAKER SAID: {text}",
154
+ provider_order=["gemini", "local"],
155
+ )
156
+ raw = (getattr(rep, "text", "") or "").strip()
157
+ raw = re.sub(r"^```(?:json)?|```$", "", raw).strip()
158
+ m = re.search(r"\{.*\}", raw, re.S)
159
+ if not m:
160
+ return None
161
+ data = json.loads(m.group(0))
162
+ kind = str(data.get("intent", "")).lower().strip()
163
+ if kind not in INTENTS:
164
+ return None
165
+ conf = float(data.get("confidence", 0.6) or 0.6)
166
+ return Intent(kind, conf, strip_address(text), "model")
167
+ except Exception:
168
+ return None
169
+
170
+
171
+ # ── Heuristic fallback ──────────────────────────────────────────────────────
172
+ def heuristic_intent(text: str) -> Intent:
173
+ """Used only when the model is unreachable.
174
+
175
+ Still deliberately broader than the old action_words list: it treats an
176
+ imperative opening as an instruction rather than requiring a listed verb.
177
+ """
178
+ body = strip_address(text) or text
179
+ if _QUESTION.match(body):
180
+ return Intent("converse", 0.55, body, "default")
181
+ # An imperative sentence usually opens with a bare verb. Rather than list
182
+ # verbs, notice what an instruction is NOT: a question, or a statement
183
+ # opening with a pronoun/article.
184
+ if re.match(r"^\s*(?:i|you|he|she|they|we|it|the|a|an|this|that|there|my)\b",
185
+ body, re.I):
186
+ return Intent("converse", 0.5, body, "default")
187
+ return Intent("act", 0.5, body, "default")
188
+
189
+
190
+ def classify(text: str, allow_model: bool = True) -> Intent:
191
+ """The one call every surface should use."""
192
+ fast = fast_intent(text)
193
+ if fast is not None:
194
+ return fast
195
+ if allow_model:
196
+ got = model_intent(text)
197
+ if got is not None:
198
+ return got
199
+ return heuristic_intent(text)
200
+
201
+
202
+ __all__ = ["classify", "Intent", "is_addressed", "strip_address",
203
+ "fast_intent", "model_intent", "heuristic_intent", "INTENTS"]
modules/omega_executor.py ADDED
@@ -0,0 +1,667 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """modules/omega_executor.py — the unlimited task engine.
2
+
3
+ WHY THIS EXISTS
4
+ ---------------
5
+ Every command path in this system used to end at a keyword table. core.commands
6
+ has an `instant_commands` dict and a long if/elif ladder; modules.task_authoring
7
+ handles exactly one application, because its app map is literally
8
+
9
+ {"notepad": ("notepad.exe", "notepad"), ...}
10
+
11
+ So "open notepad and write X then save" worked and *nothing else did*. Not
12
+ because the machine lacked the power — backend/tools/tool_registry.py already
13
+ exposes run_shell, open_app, write_file, browser control, media control and two
14
+ dozen more — but because the voice and phone paths could not reach any of it.
15
+ The powers were designed and never granted.
16
+
17
+ This module is the grant. It takes a request in plain speech, PLANS it against
18
+ the real capability catalog, and executes the plan step by step. Nothing here is
19
+ keyed to a verb list or an app name, so "restart the machine", "download this,
20
+ install it, run it ten minutes, close everything and shut down", and requests
21
+ nobody anticipated all travel the same road.
22
+
23
+ WHEN NOTHING FITS, IT WRITES THE CODE
24
+ -------------------------------------
25
+ A catalog is still a finite list, and the requirement is explicitly unlimited.
26
+ So the planner has a step kind the others don't: `python`. If no capability
27
+ matches, the model authors a short script for that step and the engine runs it,
28
+ feeding its output to the next step. That is the difference between "unlimited
29
+ within what we thought of" and unlimited.
30
+
31
+ IRREVERSIBLE ACTIONS
32
+ --------------------
33
+ Shutdown, restart, delete, format and their kin are executed — the user asked
34
+ for exactly that, in those words — but they are executed LAST in a plan and
35
+ announced first, with a cancellation window. A misheard word should not be able
36
+ to take the machine down with no way to catch it. Everything reversible runs
37
+ without ceremony.
38
+ """
39
+
40
+ from __future__ import annotations
41
+
42
+ import asyncio
43
+ import json
44
+ import os
45
+ import re
46
+ import subprocess
47
+ import sys
48
+ import tempfile
49
+ import threading
50
+ import time
51
+ from dataclasses import dataclass, field
52
+ from typing import Any, Callable, Optional
53
+
54
+ # ── Tunables ────────────────────────────────────────────────────────────────
55
+ MAX_STEPS = 40 # a chain longer than this is almost certainly a
56
+ # planner loop, not a user intention
57
+ STEP_TIMEOUT = 900 # 15 min: "run it for ten minutes" must fit
58
+ PYTHON_TIMEOUT = 300
59
+ DESTRUCTIVE_GRACE = 8 # seconds to say "cancel that" before the machine goes
60
+
61
+ _LOG_LOCK = threading.RLock()
62
+
63
+
64
+ def _log(msg: str) -> None:
65
+ with _LOG_LOCK:
66
+ try:
67
+ print(f"[omega_executor] {msg}", flush=True)
68
+ except Exception:
69
+ pass
70
+
71
+
72
+ # ── What counts as irreversible ─────────────────────────────────────────────
73
+ # Matched against the SHELL/PYTHON text actually about to run, not against the
74
+ # user's phrasing. Phrasing lies; the command does not.
75
+ _DESTRUCTIVE_PATTERNS = (
76
+ r"\bshutdown\b", r"\bstop-computer\b", r"\brestart-computer\b",
77
+ r"\blogoff\b", r"\bshutdown\.exe\b",
78
+ r"\bformat\b", r"\bdiskpart\b", r"\bmkfs\b",
79
+ r"\brd\s+/s\b", r"\brmdir\s+/s\b",
80
+ # Deletion, in any form, with or without flags.
81
+ #
82
+ # These used to require -Recurse or /q, which meant they only caught the
83
+ # deletion of whole DIRECTORY TREES. `Remove-Item -Force` on a single file
84
+ # sailed straight through unclassified — verified: a scratch file was
85
+ # permanently deleted with no deliberation and no cancel window, because
86
+ # "remove-item ... -recurse" simply did not match it.
87
+ #
88
+ # One irreplaceable file is not meaningfully less irreversible than a
89
+ # folder of them, so the flags are no longer part of the test.
90
+ r"\bremove-item\b", r"\bdel\b", r"\berase\b", r"\bunlink\b",
91
+ r"\bclear-content\b", r"\brm\s+-[a-z]*[rf]", r"\brm\s+",
92
+ r"\bcipher\s+/w\b", r"\bbcdedit\b", r"\breg\s+delete\b",
93
+ r"\bremove-partition\b", r"\bclear-disk\b",
94
+ )
95
+
96
+
97
+ def is_destructive(text: str) -> bool:
98
+ low = (text or "").lower()
99
+ return any(re.search(p, low) for p in _DESTRUCTIVE_PATTERNS)
100
+
101
+
102
+ # ── Result plumbing ─────────────────────────────────────────────────────────
103
+ @dataclass
104
+ class StepResult:
105
+ index: int
106
+ kind: str
107
+ summary: str
108
+ ok: bool
109
+ output: str = ""
110
+ error: str = ""
111
+
112
+
113
+ @dataclass
114
+ class RunResult:
115
+ ok: bool
116
+ spoken: str
117
+ steps: list[StepResult] = field(default_factory=list)
118
+ plan: list[dict] = field(default_factory=list)
119
+ cancelled: bool = False
120
+
121
+ def transcript(self) -> str:
122
+ lines = []
123
+ for s in self.steps:
124
+ mark = "ok " if s.ok else "FAIL"
125
+ lines.append(f" [{mark}] {s.index}. {s.summary}")
126
+ if s.error:
127
+ lines.append(f" {s.error[:300]}")
128
+ return "\n".join(lines)
129
+
130
+
131
+ # ── Capability catalog ──────────────────────────────────────────────────────
132
+ # Built by introspecting the REAL registry rather than by hand, so a tool added
133
+ # anywhere becomes usable by voice on the next run with no edit here. This is
134
+ # the mechanism that grants the already-designed-but-ungranted powers.
135
+ _CATALOG_CACHE: Optional[list[dict]] = None
136
+
137
+
138
+ def capability_catalog(refresh: bool = False) -> list[dict]:
139
+ global _CATALOG_CACHE
140
+ if _CATALOG_CACHE is not None and not refresh:
141
+ return _CATALOG_CACHE
142
+
143
+ caps: list[dict] = []
144
+ try:
145
+ from backend.tools.tool_registry import TOOL_REGISTRY
146
+ import inspect
147
+ for name, fn in TOOL_REGISTRY.items():
148
+ try:
149
+ sig = inspect.signature(fn)
150
+ params = [
151
+ p.name for p in sig.parameters.values()
152
+ if p.name not in ("self", "args", "kwargs")
153
+ and p.kind not in (p.VAR_POSITIONAL, p.VAR_KEYWORD)
154
+ ]
155
+ except Exception:
156
+ params = []
157
+ doc = (fn.__doc__ or "").strip().split("\n")[0][:160]
158
+ caps.append({"name": name, "params": params, "doc": doc})
159
+ except Exception as e:
160
+ _log(f"tool registry unavailable ({e}) — shell and python still work")
161
+
162
+ _CATALOG_CACHE = caps
163
+ return caps
164
+
165
+
166
+ def _catalog_text() -> str:
167
+ caps = capability_catalog()
168
+ if not caps:
169
+ return "(no named tools reachable; use shell and python)"
170
+ return "\n".join(
171
+ f"- {c['name']}({', '.join(c['params'])})"
172
+ + (f" — {c['doc']}" if c["doc"] else "")
173
+ for c in caps
174
+ )
175
+
176
+
177
+ # ── The planner ─────────────────────────────────────────────────────────────
178
+ _PLANNER_RULES = """You convert a spoken request into an executable plan for a
179
+ Windows machine. Return ONLY a JSON object, no prose, no code fences.
180
+
181
+ {"say": "<one short line to speak before starting>",
182
+ "steps": [ {"kind": "...", ...}, ... ]}
183
+
184
+ Step kinds:
185
+
186
+ {"kind":"tool","name":"<catalog name>","args":{...},"why":"<short summary>"}
187
+ Prefer this whenever a catalog tool fits.
188
+
189
+ {"kind":"shell","cmd":"<a real Windows command>","why":"..."}
190
+ PowerShell/cmd. Use for anything the catalog does not cover.
191
+
192
+ {"kind":"python","code":"<a complete python script>","why":"..."}
193
+ USE THIS WHEN NOTHING ELSE FITS. You are allowed and expected to write
194
+ new code for requests nobody anticipated. The script may import anything
195
+ installed, read $OMEGA_PREV for the previous step's output, and print its
196
+ result. Keep it self-contained.
197
+
198
+ {"kind":"wait","seconds":<number>,"why":"..."}
199
+ Real elapsed time — "run it for ten minutes" is {"kind":"wait","seconds":600}.
200
+
201
+ {"kind":"say","text":"..."}
202
+ Speak progress mid-chain.
203
+
204
+ Rules:
205
+ - Decompose chains fully and in order. "download X, install it, run it 10
206
+ minutes, close everything, shut down" is at least 5 steps, not 1.
207
+ - Irreversible steps (shutdown, restart, format, mass delete) go LAST.
208
+ - Never invent a tool name that is not in the catalog.
209
+ - Never ask the user a question; choose the sensible interpretation and act.
210
+ - Prefer one capable step over many timid ones, but never merge a wait.
211
+ """
212
+
213
+
214
+ def _extract_json(raw: str) -> Optional[dict]:
215
+ """Models wrap JSON in prose or fences no matter how firmly you ask."""
216
+ if not raw:
217
+ return None
218
+ txt = raw.strip()
219
+ txt = re.sub(r"^```(?:json)?", "", txt).strip()
220
+ txt = re.sub(r"```$", "", txt).strip()
221
+ try:
222
+ return json.loads(txt)
223
+ except Exception:
224
+ pass
225
+ # First balanced {...} block.
226
+ start = txt.find("{")
227
+ while start != -1:
228
+ depth, instr, esc = 0, False, False
229
+ for i in range(start, len(txt)):
230
+ ch = txt[i]
231
+ if esc:
232
+ esc = False
233
+ continue
234
+ if ch == "\\":
235
+ esc = True
236
+ continue
237
+ if ch == '"':
238
+ instr = not instr
239
+ continue
240
+ if instr:
241
+ continue
242
+ if ch == "{":
243
+ depth += 1
244
+ elif ch == "}":
245
+ depth -= 1
246
+ if depth == 0:
247
+ try:
248
+ return json.loads(txt[start:i + 1])
249
+ except Exception:
250
+ break
251
+ start = txt.find("{", start + 1)
252
+ return None
253
+
254
+
255
+ def plan_request(request: str, context: str = "") -> dict:
256
+ """Turn plain speech into a step plan. Never raises."""
257
+ system = _PLANNER_RULES + "\n\nCATALOG:\n" + _catalog_text()
258
+ ctx = [f"Machine: Windows. Working dir: {os.getcwd()}"]
259
+ if context:
260
+ ctx.append(context)
261
+ ctx.append(f"REQUEST: {request}")
262
+
263
+ raw = ""
264
+ try:
265
+ from core.brain_router import ask_router
266
+ rep = ask_router(
267
+ user_text=request,
268
+ system_prompt=system,
269
+ context_block="\n".join(ctx),
270
+ provider_order=["gemini", "local"],
271
+ )
272
+ raw = getattr(rep, "text", "") or ""
273
+ except Exception as e:
274
+ _log(f"planner LLM unavailable: {e}")
275
+
276
+ plan = _extract_json(raw)
277
+ if not isinstance(plan, dict) or not isinstance(plan.get("steps"), list):
278
+ # No plan means no invention: fall back to running the literal request
279
+ # as a shell line, which is still better than the old silent refusal.
280
+ return {
281
+ "say": "Running that directly.",
282
+ "steps": [{"kind": "shell", "cmd": request, "why": request}],
283
+ "degraded": True,
284
+ }
285
+
286
+ steps = [s for s in plan["steps"] if isinstance(s, dict) and s.get("kind")]
287
+ plan["steps"] = steps[:MAX_STEPS]
288
+ return plan
289
+
290
+
291
+ # ── Step execution ──────────────────────────────────────────────────────────
292
+ def _run_shell(cmd: str, timeout: int = STEP_TIMEOUT) -> tuple[bool, str, str]:
293
+ try:
294
+ creation = 0
295
+ if os.name == "nt":
296
+ creation = getattr(subprocess, "CREATE_NO_WINDOW", 0)
297
+ proc = subprocess.run(
298
+ ["powershell", "-NoProfile", "-NonInteractive", "-Command", cmd],
299
+ capture_output=True, text=True, timeout=timeout,
300
+ creationflags=creation,
301
+ )
302
+ out = (proc.stdout or "").strip()
303
+ err = (proc.stderr or "").strip()
304
+ return proc.returncode == 0, out, err
305
+ except subprocess.TimeoutExpired:
306
+ return False, "", f"timed out after {timeout}s"
307
+ except Exception as e:
308
+ return False, "", str(e)
309
+
310
+
311
+ def _run_python(code: str, prev: str = "") -> tuple[bool, str, str]:
312
+ """Run model-authored code in a child process.
313
+
314
+ A child, not exec() in-process: authored code that throws, blocks or calls
315
+ sys.exit must not be able to take JARVIS down with it.
316
+ """
317
+ path = ""
318
+ try:
319
+ fd, path = tempfile.mkstemp(suffix=".py", prefix="omega_step_")
320
+ with os.fdopen(fd, "w", encoding="utf-8") as fh:
321
+ fh.write(code)
322
+ env = dict(os.environ)
323
+ env["OMEGA_PREV"] = (prev or "")[:8000]
324
+ env["PYTHONIOENCODING"] = "utf-8"
325
+ creation = getattr(subprocess, "CREATE_NO_WINDOW", 0) if os.name == "nt" else 0
326
+ proc = subprocess.run(
327
+ [sys.executable, path], capture_output=True, text=True,
328
+ timeout=PYTHON_TIMEOUT, env=env, creationflags=creation,
329
+ )
330
+ return (proc.returncode == 0,
331
+ (proc.stdout or "").strip(),
332
+ (proc.stderr or "").strip())
333
+ except subprocess.TimeoutExpired:
334
+ return False, "", f"script timed out after {PYTHON_TIMEOUT}s"
335
+ except Exception as e:
336
+ return False, "", str(e)
337
+ finally:
338
+ if path:
339
+ try:
340
+ os.unlink(path)
341
+ except Exception:
342
+ pass
343
+
344
+
345
+ def _run_tool(name: str, args: dict) -> tuple[bool, str, str]:
346
+ try:
347
+ from backend.tools.tool_registry import TOOL_REGISTRY
348
+ except Exception as e:
349
+ return False, "", f"tool registry unavailable: {e}"
350
+ fn = TOOL_REGISTRY.get(name)
351
+ if fn is None:
352
+ return False, "", f"no such tool: {name}"
353
+ try:
354
+ res = fn(**(args or {}))
355
+ if asyncio.iscoroutine(res):
356
+ try:
357
+ asyncio.get_running_loop()
358
+ except RuntimeError:
359
+ res = asyncio.run(res)
360
+ else:
361
+ # Already inside a loop (backend request): use a private one in
362
+ # a worker thread rather than fighting the live loop.
363
+ box: dict[str, Any] = {}
364
+
365
+ def _worker():
366
+ try:
367
+ box["v"] = asyncio.run(res)
368
+ except Exception as ex:
369
+ box["e"] = ex
370
+
371
+ t = threading.Thread(target=_worker, daemon=True)
372
+ t.start()
373
+ t.join(STEP_TIMEOUT)
374
+ if "e" in box:
375
+ return False, "", str(box["e"])
376
+ res = box.get("v")
377
+ return True, ("" if res is None else str(res))[:4000], ""
378
+ except Exception as e:
379
+ return False, "", str(e)
380
+
381
+
382
+ # ── The engine ──────────────────────────────────────────────────────────────
383
+ def execute(request: str,
384
+ speak: Optional[Callable[[str], None]] = None,
385
+ context: str = "",
386
+ allow_destructive: bool = True,
387
+ confirm: Optional[Callable[[str], bool]] = None,
388
+ already_deliberated: bool = False) -> RunResult:
389
+ """Do whatever was asked. The single entry point for every surface.
390
+
391
+ `already_deliberated` is set by modules.deferred_intent when it re-runs a
392
+ held order. Without it the order would be weighed a second time here and
393
+ could be deferred again by the same reasoning that just released it — an
394
+ order could bounce between the two layers forever and never actually run.
395
+ The cancel window is still honoured; only the judgement is skipped.
396
+ """
397
+ say = speak or (lambda t: _log(f"say: {t}"))
398
+
399
+ plan = plan_request(request, context)
400
+ steps = plan.get("steps") or []
401
+ opening = (plan.get("say") or "").strip()
402
+ if opening:
403
+ say(opening)
404
+
405
+ results: list[StepResult] = []
406
+ prev_output = ""
407
+ ok_all = True
408
+
409
+ for i, step in enumerate(steps, 1):
410
+ kind = str(step.get("kind", "")).lower()
411
+ why = str(step.get("why") or step.get("text") or kind)[:180]
412
+
413
+ # Everything this step will actually run, whichever shape it arrived in.
414
+ #
415
+ # This used to be `step["cmd"] or step["code"]`, which silently exempted
416
+ # TOOL steps: the planner is free to express a deletion as
417
+ # {"kind":"tool","name":"run_shell","args":{"cmd":"Remove-Item ..."}},
418
+ # and then the command lives in args, step["cmd"] is empty, and
419
+ # is_destructive("") is False. Verified the hard way — a scratch file was
420
+ # permanently deleted with no deliberation and no cancel window because
421
+ # the plan happened to use the tool form rather than a shell step.
422
+ #
423
+ # The tool NAME matters too: run_shell carrying a shutdown is a shutdown
424
+ # regardless of how the JSON is shaped.
425
+ payload = " ".join(str(x) for x in (
426
+ step.get("cmd") or "",
427
+ step.get("code") or "",
428
+ step.get("name") or "",
429
+ json.dumps(step.get("args") or {}),
430
+ ))
431
+ if kind in ("shell", "python", "tool") and is_destructive(payload):
432
+ if not allow_destructive:
433
+ results.append(StepResult(i, kind, why, False,
434
+ error="irreversible step refused"))
435
+ ok_all = False
436
+ continue
437
+
438
+ # Think about it properly first. This is NOT a permission check —
439
+ # the order is already given and will be carried out. It is the
440
+ # judgement a person applies before pulling a plug: is something
441
+ # mid-flight that this would wreck? If so, hold and say why, and
442
+ # modules.deferred_intent will raise it again in a few minutes and
443
+ # eventually go ahead regardless. Holding is a timer, not a refusal.
444
+ if not already_deliberated:
445
+ try:
446
+ from modules.deferred_intent import deliberate, defer
447
+ verdict = deliberate(request)
448
+ if not verdict.proceed:
449
+ held = defer(request, verdict.reason)
450
+ say(verdict.speak
451
+ or f"Holding off on that — {verdict.reason}. "
452
+ f"I'll come back to it.")
453
+ results.append(StepResult(
454
+ i, kind, why, True,
455
+ output=f"deferred (#{held}): {verdict.reason}"))
456
+ return RunResult(True, "", results, steps)
457
+ except Exception as e:
458
+ _log(f"deliberation unavailable ({e}); order stands")
459
+
460
+ # The cancel window. Deliberately kept: a misheard word should have
461
+ # one last chance to be caught before the machine goes down.
462
+ say(f"{why}. Say cancel now if that's wrong.")
463
+ if confirm is not None:
464
+ if not confirm(why):
465
+ results.append(StepResult(i, kind, why, False,
466
+ error="cancelled by user"))
467
+ return RunResult(False, "Cancelled.", results, steps, True)
468
+ else:
469
+ time.sleep(DESTRUCTIVE_GRACE)
470
+
471
+ if kind == "say":
472
+ text = str(step.get("text") or why)
473
+ say(text)
474
+ results.append(StepResult(i, kind, text[:120], True))
475
+ continue
476
+
477
+ if kind == "wait":
478
+ try:
479
+ secs = float(step.get("seconds") or 0)
480
+ except Exception:
481
+ secs = 0.0
482
+ secs = max(0.0, min(secs, STEP_TIMEOUT))
483
+ results.append(StepResult(i, kind, f"waited {int(secs)}s", True))
484
+ time.sleep(secs)
485
+ continue
486
+
487
+ if kind == "tool":
488
+ good, out, err = _run_tool(str(step.get("name") or ""),
489
+ step.get("args") or {})
490
+ elif kind == "python":
491
+ good, out, err = _run_python(str(step.get("code") or ""), prev_output)
492
+ else: # shell, and anything unrecognised is treated as one
493
+ good, out, err = _run_shell(str(step.get("cmd") or ""))
494
+
495
+ # ── IMMEDIATE SELF-REPAIR ────────────────────────────────────────────
496
+ #
497
+ # A failing step used to end the chain. But most failures are not "this
498
+ # is impossible" — they are a wrong path, a missing quote, a cmdlet that
499
+ # does not exist on this box, a tool called with the wrong argument.
500
+ # A person would not abandon the task; they would look at the error, fix
501
+ # the command, and run it again, all within a few seconds.
502
+ #
503
+ # So that is what happens. The error text and the failed command go back
504
+ # to the model, which returns a corrected version, and it is retried
505
+ # immediately. The operator asked for the thing to happen NOW, and this
506
+ # is what makes it happen now rather than after a bug report.
507
+ #
508
+ # The lasting fix is a separate, slower matter — recorded and handled
509
+ # later, because stopping to redesign something mid-task is exactly the
510
+ # wrong moment for it.
511
+ if not good and kind in ("shell", "python", "tool"):
512
+ for attempt in range(1, MAX_REPAIRS + 1):
513
+ fixed = _repair_step(step, err, request)
514
+ if not fixed:
515
+ break
516
+ _log(f"step {i} repair attempt {attempt}")
517
+ if kind == "tool":
518
+ good, out, err = _run_tool(str(fixed.get("name") or ""),
519
+ fixed.get("args") or {})
520
+ elif kind == "python":
521
+ good, out, err = _run_python(str(fixed.get("code") or ""),
522
+ prev_output)
523
+ else:
524
+ good, out, err = _run_shell(str(fixed.get("cmd") or ""))
525
+ if good:
526
+ why = f"{why} (repaired)"
527
+ _remember_repair(request, step, fixed, err_before=err)
528
+ break
529
+ step = fixed # each attempt learns from the last error
530
+
531
+ results.append(StepResult(i, kind, why, good, out, err))
532
+ if out:
533
+ prev_output = out
534
+ if not good:
535
+ ok_all = False
536
+ _log(f"step {i} failed after repair attempts: {err[:200]}")
537
+ # A broken link means the rest of the chain is operating on false
538
+ # premises, so stop rather than plough on.
539
+ break
540
+
541
+ if ok_all:
542
+ spoken = _summarise_success(request, results)
543
+ else:
544
+ bad = next((r for r in results if not r.ok), None)
545
+ spoken = (f"I got through {len(results) - 1} of {len(steps)} steps, then "
546
+ f"{bad.summary.lower() if bad else 'a step'} failed."
547
+ if bad else "That didn't complete.")
548
+ return RunResult(ok_all, spoken, results, steps)
549
+
550
+
551
+ # ── Self-repair ─────────────────────────────────────────────────────────────
552
+ MAX_REPAIRS = 2 # two corrected attempts, then admit it is stuck
553
+ _REPAIR_LOG = os.path.join(
554
+ os.environ.get("JARVIS_APP_DATA_DIR") or os.path.join(
555
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))), ".friday_data"),
556
+ "repairs.json")
557
+
558
+ _REPAIR_RULES = """A command you just ran FAILED. Fix it and return the
559
+ corrected step, as JSON only, in exactly the shape you were given.
560
+
561
+ You are not redesigning anything. You are getting this ONE command to work on
562
+ this machine, right now, from the error it produced. Read the error literally:
563
+ a path that does not exist, a cmdlet that is not available, a quoting mistake,
564
+ a wrong argument name, a missing flag.
565
+
566
+ Return the same "kind" with corrected fields:
567
+ shell -> {"kind":"shell","cmd":"<fixed command>"}
568
+ python -> {"kind":"python","code":"<fixed script>"}
569
+ tool -> {"kind":"tool","name":"<tool>","args":{...}}
570
+
571
+ If the approach itself cannot work, switch to one that can — a shell step
572
+ instead of a tool, or a python step that does the job directly. Returning
573
+ something that will fail the same way is worse than useless.
574
+ """
575
+
576
+
577
+ def _repair_step(step: dict, error: str, original_request: str) -> Optional[dict]:
578
+ """Ask for a corrected version of a step that just failed."""
579
+ if not error:
580
+ return None
581
+ try:
582
+ from core.brain_router import ask_router
583
+ ctx = (f"ORIGINAL REQUEST: {original_request}\n"
584
+ f"STEP THAT FAILED: {json.dumps(step)[:1500]}\n"
585
+ f"ERROR: {error[:1200]}\n"
586
+ f"Machine: Windows. Working dir: {os.getcwd()}")
587
+ rep = ask_router(user_text="Fix this command.",
588
+ system_prompt=_REPAIR_RULES,
589
+ context_block=ctx,
590
+ provider_order=["gemini", "local"])
591
+ fixed = _extract_json(getattr(rep, "text", "") or "")
592
+ if not isinstance(fixed, dict) or not fixed.get("kind"):
593
+ return None
594
+ # A "fix" identical to what just failed is not a fix.
595
+ if (str(fixed.get("cmd") or "") == str(step.get("cmd") or "")
596
+ and str(fixed.get("code") or "") == str(step.get("code") or "")
597
+ and str(fixed.get("name") or "") == str(step.get("name") or "")):
598
+ return None
599
+ # Never let a repair quietly escalate into something irreversible.
600
+ payload = " ".join(str(x) for x in (fixed.get("cmd") or "",
601
+ fixed.get("code") or "",
602
+ fixed.get("name") or "",
603
+ json.dumps(fixed.get("args") or {})))
604
+ if is_destructive(payload) and not is_destructive(
605
+ " ".join(str(y) for y in (step.get("cmd") or "",
606
+ step.get("code") or ""))):
607
+ _log("rejected a repair that introduced an irreversible action")
608
+ return None
609
+ return fixed
610
+ except Exception as e:
611
+ _log(f"repair unavailable: {e}")
612
+ return None
613
+
614
+
615
+ def _remember_repair(request: str, broken: dict, fixed: dict,
616
+ err_before: str = "") -> None:
617
+ """Record what needed fixing, for a proper fix later.
618
+
619
+ The immediate repair gets the operator unblocked, which is the priority.
620
+ But a command that had to be corrected at runtime will need correcting
621
+ again next time unless something changes — so the evidence is kept, rather
622
+ than the problem being silently papered over on every single run.
623
+ """
624
+ try:
625
+ os.makedirs(os.path.dirname(_REPAIR_LOG), exist_ok=True)
626
+ try:
627
+ with open(_REPAIR_LOG, "r", encoding="utf-8") as fh:
628
+ data = json.load(fh)
629
+ if not isinstance(data, list):
630
+ data = []
631
+ except Exception:
632
+ data = []
633
+ data.append({
634
+ "at": time.time(),
635
+ "request": request[:300],
636
+ "broken": {k: str(v)[:400] for k, v in broken.items()},
637
+ "fixed": {k: str(v)[:400] for k, v in fixed.items()},
638
+ "error": err_before[:500],
639
+ })
640
+ with open(_REPAIR_LOG, "w", encoding="utf-8") as fh:
641
+ json.dump(data[-100:], fh, indent=2)
642
+ except Exception:
643
+ pass
644
+
645
+
646
+ def pending_repairs() -> list[dict]:
647
+ """Runtime corrections so far — the queue for real, lasting fixes."""
648
+ try:
649
+ with open(_REPAIR_LOG, "r", encoding="utf-8") as fh:
650
+ data = json.load(fh)
651
+ return data if isinstance(data, list) else []
652
+ except Exception:
653
+ return []
654
+
655
+
656
+ def _summarise_success(request: str, results: list[StepResult]) -> str:
657
+ done = [r for r in results if r.ok and r.kind not in ("say",)]
658
+ if not done:
659
+ return "Nothing to do there."
660
+ if len(done) == 1:
661
+ return f"Done — {done[0].summary}."
662
+ return f"Done. {len(done)} steps: " + "; ".join(r.summary for r in done[:4]) + (
663
+ "…" if len(done) > 4 else ".")
664
+
665
+
666
+ __all__ = ["execute", "plan_request", "capability_catalog", "is_destructive",
667
+ "RunResult", "StepResult"]
modules/process_utils.py CHANGED
@@ -58,3 +58,47 @@ def run_no_window(
58
  kwargs["creationflags"] = windows_creationflags_no_window()
59
  return subprocess.run(list(args), **kwargs)
60
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
58
  kwargs["creationflags"] = windows_creationflags_no_window()
59
  return subprocess.run(list(args), **kwargs)
60
 
61
+
62
+ _console_suppressed = False
63
+
64
+
65
+ def suppress_console_windows() -> None:
66
+ """Stop every child process in THIS process from flashing a console.
67
+
68
+ The helpers above only work where someone remembered to call them, and
69
+ across modules/ there are around forty bare `subprocess.run`/`Popen` calls
70
+ that do not. Most are harmless one-offs, but the watch loops shell out
71
+ constantly — `netstat -ano` for intrusion and connection checks, PowerShell
72
+ for disk health and driver anomalies, `wmic` for process listings — every
73
+ few seconds, forever.
74
+
75
+ Each of those is a CONSOLE program. Launched from a windowless parent
76
+ (pythonw, which is how the daemon, the relay and the tray all run), Windows
77
+ has no console to attach them to, so it creates one — and the window
78
+ appears and vanishes. That is the flashing: not one bug, one per call site.
79
+
80
+ Fixing forty call sites individually invites the forty-first. This sets the
81
+ default once, for the whole process: any child created WITHOUT explicit
82
+ creationflags gets CREATE_NO_WINDOW. A caller that passes its own flags is
83
+ left completely alone, so nothing that deliberately wants a window loses it.
84
+
85
+ Call this early in any entry point that runs without a console.
86
+ """
87
+ global _console_suppressed
88
+ if _console_suppressed or os.name != "nt":
89
+ return
90
+
91
+ CREATE_NO_WINDOW = 0x08000000
92
+ _original_init = subprocess.Popen.__init__
93
+
94
+ def _init(self, *args, **kwargs):
95
+ # Only fill in a default. Never override an explicit choice, and never
96
+ # add DETACHED_PROCESS here — that would break callers who read the
97
+ # child's stdout, which several of these modules do.
98
+ if not kwargs.get("creationflags"):
99
+ kwargs["creationflags"] = CREATE_NO_WINDOW
100
+ return _original_init(self, *args, **kwargs)
101
+
102
+ subprocess.Popen.__init__ = _init
103
+ _console_suppressed = True
104
+
modules/security_review.py CHANGED
@@ -114,8 +114,19 @@ def run_security_review(target: str = None) -> str:
114
  source_context = _gather_target_context(target_path)
115
 
116
  # Build the final prompt
 
 
 
 
 
 
 
 
 
 
 
117
  prompt = (
118
- f"You are JARVIS, conducting an exhaustive security audit based on the provided framework.\n\n"
119
  f"Target System Scope:\n{target_path}\n\n"
120
  f"Discovered Source Code & Configurations:\n{source_context}\n\n"
121
  f"Please begin your analysis and output the final markdown report."
 
114
  source_context = _gather_target_context(target_path)
115
 
116
  # Build the final prompt
117
+ #
118
+ # The persona is read, not assumed: this said "You are JARVIS" outright,
119
+ # so a security review run while FRIDAY was on duty was conducted in the
120
+ # other assistant's name. The audit is the same either way, but the
121
+ # report it writes is spoken back to the user in first person.
122
+ try:
123
+ from modules.assistant_identity import get_assistant_name
124
+ _who = get_assistant_name()
125
+ except Exception:
126
+ _who = "JARVIS"
127
+
128
  prompt = (
129
+ f"You are {_who}, conducting an exhaustive security audit based on the provided framework.\n\n"
130
  f"Target System Scope:\n{target_path}\n\n"
131
  f"Discovered Source Code & Configurations:\n{source_context}\n\n"
132
  f"Please begin your analysis and output the final markdown report."
modules/task_authoring.py ADDED
@@ -0,0 +1,360 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ """Multi-step desktop tasks: open an app, type into it, save the file.
4
+
5
+ WHY THIS EXISTS
6
+ ---------------
7
+ "open notepad and write JARVIS then save to desktop" used to produce:
8
+
9
+ Executing 2 linked tasks.
10
+ Launching notepad and write jarvis. <- one app name, not two steps
11
+ Got it: save to desktop. Tell me the outcome you want.
12
+
13
+ The chain splitter in core.commands splits on "then" only, so "open notepad and
14
+ write JARVIS" stayed a single instruction and was handed to the app launcher as
15
+ if "notepad and write jarvis" were a program. Nothing could type, and nothing
16
+ could save — there was no handler for either verb.
17
+
18
+ This module does the three things that sentence actually asks for, in order,
19
+ against the real window: launch, type, save-as. It is deliberately narrow. It
20
+ does NOT try to be a general agent; it recognises a small, explicit vocabulary
21
+ and refuses anything outside it rather than guessing and reporting success.
22
+
23
+ WHY UI AUTOMATION AND NOT A FILE WRITE
24
+ --------------------------------------
25
+ "open notepad and write X then save" could be faked by writing a .txt directly
26
+ and never opening anything. That would pass a naive test and be a lie: the user
27
+ asked to SEE it happen, and the same mechanism has to work for editors that have
28
+ no file format we can synthesise. So this drives the real window with real
29
+ keystrokes.
30
+ """
31
+
32
+ import os
33
+ import re
34
+ import subprocess
35
+ import time
36
+ from dataclasses import dataclass, field
37
+ from pathlib import Path
38
+
39
+ try:
40
+ import keyboard # type: ignore
41
+ except Exception: # pragma: no cover - depends on host packages
42
+ keyboard = None
43
+
44
+ try:
45
+ import win32gui # type: ignore
46
+ except Exception: # pragma: no cover
47
+ win32gui = None
48
+
49
+
50
+ # Programs we know how to launch by a spoken name, and the window title
51
+ # fragment that proves the right one came up.
52
+ KNOWN_APPS: dict[str, tuple[str, str]] = {
53
+ "notepad": ("notepad.exe", "notepad"),
54
+ "wordpad": ("write.exe", "wordpad"),
55
+ "word": ("winword.exe", "word"),
56
+ "excel": ("excel.exe", "excel"),
57
+ "paint": ("mspaint.exe", "paint"),
58
+ "calculator": ("calc.exe", "calculator"),
59
+ "explorer": ("explorer.exe", "explorer"),
60
+ "chrome": ("chrome.exe", "chrome"),
61
+ "edge": ("msedge.exe", "edge"),
62
+ }
63
+
64
+ # Where "save to X" can mean.
65
+ KNOWN_FOLDERS: dict[str, Path] = {
66
+ "desktop": Path.home() / "Desktop",
67
+ "documents": Path.home() / "Documents",
68
+ "downloads": Path.home() / "Downloads",
69
+ "pictures": Path.home() / "Pictures",
70
+ }
71
+
72
+
73
+ @dataclass
74
+ class Step:
75
+ kind: str # "open" | "type" | "save"
76
+ value: str = ""
77
+ folder: Path | None = None
78
+ filename: str | None = None
79
+
80
+
81
+ @dataclass
82
+ class TaskResult:
83
+ ok: bool
84
+ steps_done: int
85
+ messages: list[str] = field(default_factory=list)
86
+
87
+ @property
88
+ def message(self) -> str:
89
+ return " ".join(self.messages) if self.messages else ""
90
+
91
+
92
+ # --------------------------------------------------------------------------- #
93
+ # parsing
94
+ # --------------------------------------------------------------------------- #
95
+ def _split_clauses(text: str) -> list[str]:
96
+ """Break on the connectives people actually use, not just "then".
97
+
98
+ core.commands.split_chain only knows "then", which is why "open notepad and
99
+ write JARVIS" survived as one clause. Splitting on "and"/","/"then" as
100
+ SEPARATORS BETWEEN VERBS keeps "write hello and goodbye" intact, because the
101
+ split only happens where the next clause starts with a known verb.
102
+ """
103
+ parts = re.split(r"\s*(?:,|\bthen\b|\band\b|\bafter that\b)\s*", text, flags=re.I)
104
+ verbs = ("open", "launch", "start", "run", "write", "type", "enter", "save", "store")
105
+ clauses: list[str] = []
106
+ for raw in parts:
107
+ piece = raw.strip()
108
+ if not piece:
109
+ continue
110
+ first = piece.split()[0].lower()
111
+ if first in verbs or not clauses:
112
+ clauses.append(piece)
113
+ else:
114
+ # A fragment that does not begin a new action belongs to the one
115
+ # before it: "write hello" + "goodbye" -> "write hello and goodbye".
116
+ clauses[-1] = f"{clauses[-1]} and {piece}"
117
+ return clauses
118
+
119
+
120
+ def parse_task(text: str) -> list[Step]:
121
+ """Turn a sentence into ordered steps. Returns [] when it is not our shape."""
122
+ steps: list[Step] = []
123
+ for clause in _split_clauses(text or ""):
124
+ low = clause.lower().strip()
125
+
126
+ m = re.match(r"^(?:open|launch|start|run)\s+(.+)$", low)
127
+ if m:
128
+ target = m.group(1).strip()
129
+ # Longest known app name that appears, so "open notepad app" works.
130
+ app = next((name for name in sorted(KNOWN_APPS, key=len, reverse=True)
131
+ if name in target), None)
132
+ steps.append(Step("open", app or target))
133
+ continue
134
+
135
+ m = re.match(r"^(?:write|type|enter)\s+(.+)$", clause, flags=re.I)
136
+ if m:
137
+ payload = m.group(1).strip()
138
+ # Keep the user's capitalisation: they asked for "JARVIS", not "jarvis".
139
+ payload = payload.strip("\"'")
140
+ steps.append(Step("type", payload))
141
+ continue
142
+
143
+ m = re.match(r"^(?:save|store)\b(.*)$", low)
144
+ if m:
145
+ rest = m.group(1)
146
+ folder = None
147
+ for name, path in KNOWN_FOLDERS.items():
148
+ if name in rest:
149
+ folder = path
150
+ break
151
+ fname = None
152
+ fm = re.search(r"\bas\s+([\w\-. ]+)$", rest)
153
+ if fm:
154
+ fname = fm.group(1).strip()
155
+ steps.append(Step("save", rest.strip(), folder, fname))
156
+ continue
157
+
158
+ # An unrecognised clause makes the whole task unsafe to guess at.
159
+ return []
160
+ return steps
161
+
162
+
163
+ # --------------------------------------------------------------------------- #
164
+ # execution
165
+ # --------------------------------------------------------------------------- #
166
+ def _foreground_title() -> str:
167
+ if win32gui is None:
168
+ return ""
169
+ try:
170
+ return win32gui.GetWindowText(win32gui.GetForegroundWindow()) or ""
171
+ except Exception:
172
+ return ""
173
+
174
+
175
+ def _wait_for_window(fragment: str, timeout: float = 12.0) -> bool:
176
+ """Wait until the app we launched is actually in front.
177
+
178
+ Typing before the window exists sends the keystrokes to whatever WAS in
179
+ front — which is how a naive version of this ends up typing into the user's
180
+ browser or, worse, a terminal.
181
+ """
182
+ deadline = time.time() + timeout
183
+ fragment = fragment.lower()
184
+ while time.time() < deadline:
185
+ if fragment in _foreground_title().lower():
186
+ return True
187
+ time.sleep(0.35)
188
+ return False
189
+
190
+
191
+ SAVE_DIALOG_TITLES = ("save as", "save", "guardar", "enregistrer")
192
+
193
+
194
+ def _wait_for_save_dialog(timeout: float = 8.0) -> bool:
195
+ """Block until a Save dialog owns the foreground window.
196
+
197
+ The editor's own title (for example "Untitled - Notepad") is explicitly NOT
198
+ a match: seeing it means the dialog has not opened yet and anything typed
199
+ would land in the document.
200
+ """
201
+ deadline = time.time() + timeout
202
+ while time.time() < deadline:
203
+ title = _foreground_title().lower()
204
+ if title and any(t in title for t in SAVE_DIALOG_TITLES):
205
+ return True
206
+ time.sleep(0.25)
207
+ return False
208
+
209
+
210
+ def _type_text(text: str) -> bool:
211
+ if keyboard is None:
212
+ return False
213
+ try:
214
+ # write() sends unicode directly rather than emulating scan codes, so
215
+ # punctuation and layout differences do not corrupt the text.
216
+ keyboard.write(text, delay=0.01)
217
+ return True
218
+ except Exception:
219
+ return False
220
+
221
+
222
+ def run_task(text: str, speak=None) -> TaskResult:
223
+ """Execute a parsed multi-step task. Never reports a step it did not do."""
224
+ # Speak EVERY message as it happens, not in a batch at the end.
225
+ #
226
+ # The first version collected messages and only spoke them on the success
227
+ # path, so every failure returned with an empty `spoken` list and the caller
228
+ # fell back to "I don't have a handler for that yet" — hiding the actual
229
+ # reason the task stopped. A progress report that only appears when nothing
230
+ # went wrong is worse than none.
231
+ def say(msg: str) -> None:
232
+ if speak and msg:
233
+ speak(msg)
234
+
235
+ steps = parse_task(text)
236
+ if not steps:
237
+ return TaskResult(False, 0, ["I couldn't turn that into steps I can run."])
238
+ if keyboard is None or win32gui is None:
239
+ return TaskResult(False, 0,
240
+ ["Desktop automation isn't available on this machine."])
241
+
242
+ messages: list[str] = []
243
+ def note(msg: str) -> None:
244
+ messages.append(msg)
245
+ say(msg)
246
+ done = 0
247
+ opened_app: str | None = None
248
+
249
+ for step in steps:
250
+ if step.kind == "open":
251
+ exe, title_fragment = KNOWN_APPS.get(step.value, (step.value, step.value))
252
+ try:
253
+ subprocess.Popen(
254
+ ["cmd.exe", "/c", "start", "", exe],
255
+ stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
256
+ creationflags=getattr(subprocess, "CREATE_NO_WINDOW", 0))
257
+ except Exception as exc:
258
+ note(f"Couldn't open {step.value}: {exc}")
259
+ return TaskResult(False, done, messages)
260
+ # NEVER TYPE INTO A DOCUMENT THAT ALREADY HAS CONTENT.
261
+ #
262
+ # Windows 11 Notepad restores its previous session, so launching it
263
+ # can bring back the user's OWN unsaved work — and this code then
264
+ # typed into that document and pressed Ctrl+S, which saved silently
265
+ # because the tab already had a path. Verified happening: a tab
266
+ # holding a personal document was modified by a test run.
267
+ #
268
+ # Ctrl+N forces a brand-new empty document first. An editor that does
269
+ # not support it simply ignores the keystroke, so this is safe to
270
+ # send unconditionally — and infinitely safer than assuming a fresh
271
+ # window.
272
+ if _wait_for_window(title_fragment) and step.value in ("notepad", "wordpad", "word"):
273
+ try:
274
+ keyboard.send("ctrl+n")
275
+ time.sleep(1.0)
276
+ except Exception:
277
+ pass
278
+
279
+ if not _wait_for_window(title_fragment):
280
+ messages.append(
281
+ f"Opened {step.value}, but its window never came to the front — "
282
+ "stopping so I don't type into the wrong place.")
283
+ return TaskResult(False, done, messages)
284
+ opened_app = step.value
285
+ note(f"Opened {step.value}.")
286
+ done += 1
287
+
288
+ elif step.kind == "type":
289
+ if opened_app is None:
290
+ note("Nothing is open to type into.")
291
+ return TaskResult(False, done, messages)
292
+ time.sleep(0.4)
293
+ if not _type_text(step.value):
294
+ note("Couldn't send the text.")
295
+ return TaskResult(False, done, messages)
296
+ note(f'Typed "{step.value}".')
297
+ done += 1
298
+
299
+ elif step.kind == "save":
300
+ folder = step.folder or (Path.home() / "Desktop")
301
+ name = step.filename or f"jarvis-{time.strftime('%H%M%S')}.txt"
302
+ if not os.path.splitext(name)[1]:
303
+ name += ".txt"
304
+ target = folder / name
305
+ try:
306
+ folder.mkdir(parents=True, exist_ok=True)
307
+ except Exception:
308
+ pass
309
+ try:
310
+ keyboard.send("ctrl+s")
311
+ # WAIT FOR THE DIALOG, don't guess at a sleep.
312
+ #
313
+ # A fixed 1.2s worked when run from an interactive console and
314
+ # failed from the relay's background process, where the dialog
315
+ # takes longer to paint. The path was then typed into the
316
+ # DOCUMENT instead of the filename box and Enter just added a
317
+ # newline — the file never existed, and the only clue was a
318
+ # message the caller never saw. Same rule as the app window
319
+ # above: prove the target has focus before sending keys to it.
320
+ if not _wait_for_save_dialog():
321
+ note("The save dialog never appeared, so nothing was typed "
322
+ "into it — your text is still in the editor, unsaved.")
323
+ return TaskResult(False, done, messages)
324
+ # The Save dialog focuses its filename box; a full path in that
325
+ # box is honoured by the common dialog, so no navigation is
326
+ # needed and the file lands exactly where asked.
327
+ if not _type_text(str(target)):
328
+ note("Couldn't fill in the save dialog.")
329
+ return TaskResult(False, done, messages)
330
+ time.sleep(0.4)
331
+ keyboard.send("enter")
332
+ # Give the write a moment, then poll rather than assume.
333
+ for _ in range(20):
334
+ if target.exists():
335
+ break
336
+ time.sleep(0.25)
337
+ except Exception as exc:
338
+ note(f"Save failed: {exc}")
339
+ return TaskResult(False, done, messages)
340
+
341
+ # VERIFY. A save dialog can be refused, redirected or silently
342
+ # cancelled; the only proof is the file being on disk.
343
+ if target.exists():
344
+ note(f"Saved to {target}.")
345
+ done += 1
346
+ else:
347
+ # note(), not messages.append(): appended text is never spoken,
348
+ # so this exact reason was collected and thrown away — the caller
349
+ # saw an empty result and fell back to "no handler".
350
+ note(f"I pressed save, but {target} isn't on disk. Your text is "
351
+ "still open in the editor.")
352
+ return TaskResult(False, done, messages)
353
+
354
+ return TaskResult(True, done, messages)
355
+
356
+
357
+ def looks_like_task(text: str) -> bool:
358
+ """Cheap gate for the dispatcher: at least two steps, one of them an action."""
359
+ steps = parse_task(text)
360
+ return len(steps) >= 2 and any(s.kind in ("type", "save") for s in steps)