Spaces:
Running on Zero
Running on Zero
Serve Figment v14p through HF ZeroGPU
Browse files- .env.example +14 -0
- README.md +7 -4
- app.py +3 -1
- figment/config.py +19 -5
- figment/model_client.py +8 -0
- figment/trace.py +1 -1
- figment/zerogpu_runtime.py +147 -0
- requirements.txt +12 -0
- tests/test_model_client.py +32 -0
- tests/test_runtime_honesty.py +23 -4
.env.example
CHANGED
|
@@ -15,6 +15,12 @@ HF_ENDPOINT_URL=
|
|
| 15 |
OMNI_ENDPOINT_URL=
|
| 16 |
HF_TOKEN=
|
| 17 |
LLAMA_BASE_URL=http://127.0.0.1:8001/v1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
FIGMENT_TRACE_DIR=traces
|
| 19 |
FIGMENT_SMOKE_ALLOW_NETWORK=false
|
| 20 |
FIGMENT_SMOKE_TIMEOUT_SECONDS=8
|
|
@@ -27,6 +33,14 @@ FIGMENT_SMOKE_TRACE_PATH=
|
|
| 27 |
# AUDIO_BACKEND=omni_native
|
| 28 |
# ENABLE_AUDIO_INTAKE=true
|
| 29 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 30 |
# Local/offline proof. This can use the smaller local model, or self-hosted Omni
|
| 31 |
# on adequate local hardware; smoke output records the configured LOCAL_MODEL_ID.
|
| 32 |
# FIGMENT_MODE=local
|
|
|
|
| 15 |
OMNI_ENDPOINT_URL=
|
| 16 |
HF_TOKEN=
|
| 17 |
LLAMA_BASE_URL=http://127.0.0.1:8001/v1
|
| 18 |
+
ZEROGPU_MODEL_REPO=build-small-hackathon/figment-finetuned-model-archive
|
| 19 |
+
ZEROGPU_MODEL_SUBFOLDER=figment_sft_v14p/figment-sft-v14p-lora-merged-bf16
|
| 20 |
+
ZEROGPU_DURATION_SECONDS=120
|
| 21 |
+
ZEROGPU_SIZE=large
|
| 22 |
+
ZEROGPU_MAX_CONTEXT_TOKENS=16384
|
| 23 |
+
ZEROGPU_MAX_GENERATION_TOKENS=1536
|
| 24 |
FIGMENT_TRACE_DIR=traces
|
| 25 |
FIGMENT_SMOKE_ALLOW_NETWORK=false
|
| 26 |
FIGMENT_SMOKE_TIMEOUT_SECONDS=8
|
|
|
|
| 33 |
# AUDIO_BACKEND=omni_native
|
| 34 |
# ENABLE_AUDIO_INTAKE=true
|
| 35 |
|
| 36 |
+
# Public HF ZeroGPU Space route for the published v14p model:
|
| 37 |
+
# FIGMENT_MODE=hosted
|
| 38 |
+
# MODEL_STACK=local_4b_parakeet
|
| 39 |
+
# MODEL_BACKEND=hf_zerogpu
|
| 40 |
+
# LOCAL_MODEL_ID=figment-sft-v14p-lora-merged-bf16
|
| 41 |
+
# AUDIO_BACKEND=none
|
| 42 |
+
# ENABLE_AUDIO_INTAKE=false
|
| 43 |
+
|
| 44 |
# Local/offline proof. This can use the smaller local model, or self-hosted Omni
|
| 45 |
# on adequate local hardware; smoke output records the configured LOCAL_MODEL_ID.
|
| 46 |
# FIGMENT_MODE=local
|
README.md
CHANGED
|
@@ -7,7 +7,9 @@ sdk: gradio
|
|
| 7 |
sdk_version: 6.17.3
|
| 8 |
app_file: app.py
|
| 9 |
pinned: false
|
| 10 |
-
python_version: 3.
|
|
|
|
|
|
|
| 11 |
---
|
| 12 |
|
| 13 |
# Figment
|
|
@@ -22,7 +24,7 @@ Figment turns messy field intake into a card-cited protocol workflow: confirm th
|
|
| 22 |
|
| 23 |
| Surface | Current evidence | What it means | Boundary |
|
| 24 |
| --- | --- | --- | --- |
|
| 25 |
-
| Public Space | [build-small-hackathon/figment](https://huggingface.co/spaces/build-small-hackathon/figment) is `
|
| 26 |
| Hosted Omni eval | `31/50` whole-output competence, `8/50` full fallback, `480/650` model-retained fields, `170/650` deterministic patches, and `50/50` final validation. | Hosted Omni can carry bounded fields, and the app can keep outputs inside the safety contract. | `50/50` final validation is app safety after validation, repair, and fallback. It is not pure model performance. |
|
| 27 |
| 4B LoRA system eval | v14p repair-union on the corrected 150-case field-workflow holdout: `150/150` competence, `150/150` expected labels, `150/150` final validation, `0` deterministic patches, `0` fallback. Raw first-pass success is `146/150`; `4/150` cases close through focused model repair. | The strongest documented small-model result is model-owned output plus model repair on a synthetic/de-identified holdout. | This is not clinical validation, target-user validation, local ASR proof, or proof that raw first-pass output solved every case. |
|
| 28 |
| Public artifacts | [model archive](https://huggingface.co/build-small-hackathon/figment-finetuned-model-archive) and [eval/training dataset](https://huggingface.co/datasets/build-small-hackathon/figment-eval-traces). | Versioned BF16/GGUF model artifacts, synthetic corpora, eval traces, and summaries are inspectable outside this checkout. | Generated `traces/`, `data/finetune/`, weights, and checkpoint folders are intentionally not part of a clean clone. |
|
|
@@ -69,7 +71,7 @@ app.py
|
|
| 69 |
-> figment/rules.py deterministic danger-sign rules
|
| 70 |
-> figment/retrieval.py local protocol-card retrieval
|
| 71 |
-> figment/prompt_builder.py bounded navigator prompt
|
| 72 |
-
-> figment/model_client.py hosted Omni, local OpenAI-compatible, or canned route
|
| 73 |
-> figment/navigator.py raw output, scaffold, repair, fallback orchestration
|
| 74 |
-> figment/validators.py schema, citations, urgency floor, safety checks
|
| 75 |
-> figment/field_provenance.py model_raw / model_repaired / deterministic_fallback labels
|
|
@@ -87,12 +89,13 @@ The safety pattern is deliberate:
|
|
| 87 |
|
| 88 |
## Models
|
| 89 |
|
| 90 |
-
Figment supports
|
| 91 |
|
| 92 |
| Route | Backend | Use |
|
| 93 |
| --- | --- | --- |
|
| 94 |
| Canned fallback | `MODEL_BACKEND=canned` | No-secret app smoke, UI development, honest fallback traces. |
|
| 95 |
| Hosted Omni | `MODEL_BACKEND=hosted_omni` with `NVIDIA_API_KEY` | Live hosted demo and hosted eval path using `nvidia/nemotron-3-nano-omni-30b-a3b-reasoning`. |
|
|
|
|
| 96 |
| Local OpenAI-compatible | `MODEL_BACKEND=llama_cpp` with `LLAMA_BASE_URL` | Local text-navigation route for the 4B BF16/GGUF artifacts and local evidence bundles. |
|
| 97 |
|
| 98 |
The Build Small constraint is <=32B total parameters. The hosted Omni path is tracked with a parameter-count caveat: the NVIDIA model-card body reports 31B total parameters, while sidebar counts have differed. The 4B BF16 base model, `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`, is the local text-navigation training target.
|
|
|
|
| 7 |
sdk_version: 6.17.3
|
| 8 |
app_file: app.py
|
| 9 |
pinned: false
|
| 10 |
+
python_version: 3.12.12
|
| 11 |
+
preload_from_hub:
|
| 12 |
+
- build-small-hackathon/figment-finetuned-model-archive figment_sft_v14p/figment-sft-v14p-lora-merged-bf16/chat_template.jinja,figment_sft_v14p/figment-sft-v14p-lora-merged-bf16/config.json,figment_sft_v14p/figment-sft-v14p-lora-merged-bf16/configuration_nemotron_h.py,figment_sft_v14p/figment-sft-v14p-lora-merged-bf16/generation_config.json,figment_sft_v14p/figment-sft-v14p-lora-merged-bf16/model-00001-of-00002.safetensors,figment_sft_v14p/figment-sft-v14p-lora-merged-bf16/model-00002-of-00002.safetensors,figment_sft_v14p/figment-sft-v14p-lora-merged-bf16/model.safetensors.index.json,figment_sft_v14p/figment-sft-v14p-lora-merged-bf16/modeling_nemotron_h.py,figment_sft_v14p/figment-sft-v14p-lora-merged-bf16/special_tokens_map.json,figment_sft_v14p/figment-sft-v14p-lora-merged-bf16/tokenizer.json,figment_sft_v14p/figment-sft-v14p-lora-merged-bf16/tokenizer_config.json
|
| 13 |
---
|
| 14 |
|
| 15 |
# Figment
|
|
|
|
| 24 |
|
| 25 |
| Surface | Current evidence | What it means | Boundary |
|
| 26 |
| --- | --- | --- | --- |
|
| 27 |
+
| Public Space | [build-small-hackathon/figment](https://huggingface.co/spaces/build-small-hackathon/figment) is configured for `MODEL_BACKEND=hf_zerogpu` against the published v14p BF16 artifact. | The public route can call the same 4B v14p model archive used in the strongest small-model evidence bundle. | ZeroGPU has queue/quota limits; live route proof should include a fresh Space call and trace route. |
|
| 28 |
| Hosted Omni eval | `31/50` whole-output competence, `8/50` full fallback, `480/650` model-retained fields, `170/650` deterministic patches, and `50/50` final validation. | Hosted Omni can carry bounded fields, and the app can keep outputs inside the safety contract. | `50/50` final validation is app safety after validation, repair, and fallback. It is not pure model performance. |
|
| 29 |
| 4B LoRA system eval | v14p repair-union on the corrected 150-case field-workflow holdout: `150/150` competence, `150/150` expected labels, `150/150` final validation, `0` deterministic patches, `0` fallback. Raw first-pass success is `146/150`; `4/150` cases close through focused model repair. | The strongest documented small-model result is model-owned output plus model repair on a synthetic/de-identified holdout. | This is not clinical validation, target-user validation, local ASR proof, or proof that raw first-pass output solved every case. |
|
| 30 |
| Public artifacts | [model archive](https://huggingface.co/build-small-hackathon/figment-finetuned-model-archive) and [eval/training dataset](https://huggingface.co/datasets/build-small-hackathon/figment-eval-traces). | Versioned BF16/GGUF model artifacts, synthetic corpora, eval traces, and summaries are inspectable outside this checkout. | Generated `traces/`, `data/finetune/`, weights, and checkpoint folders are intentionally not part of a clean clone. |
|
|
|
|
| 71 |
-> figment/rules.py deterministic danger-sign rules
|
| 72 |
-> figment/retrieval.py local protocol-card retrieval
|
| 73 |
-> figment/prompt_builder.py bounded navigator prompt
|
| 74 |
+
-> figment/model_client.py hosted Omni, HF ZeroGPU, local OpenAI-compatible, or canned route
|
| 75 |
-> figment/navigator.py raw output, scaffold, repair, fallback orchestration
|
| 76 |
-> figment/validators.py schema, citations, urgency floor, safety checks
|
| 77 |
-> figment/field_provenance.py model_raw / model_repaired / deterministic_fallback labels
|
|
|
|
| 89 |
|
| 90 |
## Models
|
| 91 |
|
| 92 |
+
Figment supports four runtime routes:
|
| 93 |
|
| 94 |
| Route | Backend | Use |
|
| 95 |
| --- | --- | --- |
|
| 96 |
| Canned fallback | `MODEL_BACKEND=canned` | No-secret app smoke, UI development, honest fallback traces. |
|
| 97 |
| Hosted Omni | `MODEL_BACKEND=hosted_omni` with `NVIDIA_API_KEY` | Live hosted demo and hosted eval path using `nvidia/nemotron-3-nano-omni-30b-a3b-reasoning`. |
|
| 98 |
+
| HF ZeroGPU v14p | `MODEL_BACKEND=hf_zerogpu` with `ZEROGPU_MODEL_REPO` / `ZEROGPU_MODEL_SUBFOLDER` | Public Space route using the published v14p BF16 merged model on Hugging Face ZeroGPU. |
|
| 99 |
| Local OpenAI-compatible | `MODEL_BACKEND=llama_cpp` with `LLAMA_BASE_URL` | Local text-navigation route for the 4B BF16/GGUF artifacts and local evidence bundles. |
|
| 100 |
|
| 101 |
The Build Small constraint is <=32B total parameters. The hosted Omni path is tracked with a parameter-count caveat: the NVIDIA model-card body reports 31B total parameters, while sidebar counts have differed. The 4B BF16 base model, `nvidia/NVIDIA-Nemotron-3-Nano-4B-BF16`, is the local text-navigation training target.
|
app.py
CHANGED
|
@@ -344,7 +344,7 @@ def _app_header_html() -> str:
|
|
| 344 |
|
| 345 |
def _statusline_html(config: FigmentConfig) -> str:
|
| 346 |
audio_chip = "green" if config.enable_audio_intake else "amber"
|
| 347 |
-
backend_chip = "blue" if config.model_backend
|
| 348 |
return f"""
|
| 349 |
<div class="figment-statusline">
|
| 350 |
<strong>Runtime</strong>
|
|
@@ -955,6 +955,8 @@ __FIGMENT_CSS__
|
|
| 955 |
def _model_mode_label(config: FigmentConfig) -> str:
|
| 956 |
if config.model_backend == "hosted_omni":
|
| 957 |
return "Configured backend: hosted_omni"
|
|
|
|
|
|
|
| 958 |
if config.model_backend == "llama_cpp":
|
| 959 |
return "Configured backend: llama_cpp"
|
| 960 |
return "Configured backend: canned"
|
|
|
|
| 344 |
|
| 345 |
def _statusline_html(config: FigmentConfig) -> str:
|
| 346 |
audio_chip = "green" if config.enable_audio_intake else "amber"
|
| 347 |
+
backend_chip = "blue" if config.model_backend in {"hosted_omni", "hf_zerogpu"} else "amber"
|
| 348 |
return f"""
|
| 349 |
<div class="figment-statusline">
|
| 350 |
<strong>Runtime</strong>
|
|
|
|
| 955 |
def _model_mode_label(config: FigmentConfig) -> str:
|
| 956 |
if config.model_backend == "hosted_omni":
|
| 957 |
return "Configured backend: hosted_omni"
|
| 958 |
+
if config.model_backend == "hf_zerogpu":
|
| 959 |
+
return "Configured backend: hf_zerogpu"
|
| 960 |
if config.model_backend == "llama_cpp":
|
| 961 |
return "Configured backend: llama_cpp"
|
| 962 |
return "Configured backend: canned"
|
figment/config.py
CHANGED
|
@@ -15,9 +15,12 @@ PARAKEET_ASR_MODEL_ID = "nvidia/parakeet-rnnt-1.1b"
|
|
| 15 |
NVIDIA_OMNI_API_MODEL_ID = "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning"
|
| 16 |
NVIDIA_API_BASE_URL = "https://integrate.api.nvidia.com/v1"
|
| 17 |
FIGMENT_CANNED_MODEL_ID = "figment-canned-deterministic"
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
MODEL_STACKS = {"omni_native", "local_4b_parakeet"}
|
| 20 |
-
MODEL_BACKENDS = {"hosted_omni", "llama_cpp", "canned"}
|
| 21 |
AUDIO_BACKENDS = {"omni_native", "parakeet_nemo", "canned", "none"}
|
| 22 |
FIGMENT_MODES = {"hosted", "local", "canned"}
|
| 23 |
|
|
@@ -86,6 +89,8 @@ class FigmentConfig:
|
|
| 86 |
omni_endpoint_url: str = ""
|
| 87 |
hf_endpoint_url: str = ""
|
| 88 |
llama_base_url: str = "http://127.0.0.1:8001/v1"
|
|
|
|
|
|
|
| 89 |
trace_dir: Path = Path("traces")
|
| 90 |
|
| 91 |
@classmethod
|
|
@@ -109,7 +114,7 @@ class FigmentConfig:
|
|
| 109 |
backend = {"hosted": "hosted_omni", "local": "llama_cpp", "canned": "canned"}[backend]
|
| 110 |
stack = os.getenv("MODEL_STACK", "").strip()
|
| 111 |
if not stack:
|
| 112 |
-
stack = "local_4b_parakeet" if backend
|
| 113 |
audio = os.getenv("AUDIO_BACKEND", "none").strip() or "none"
|
| 114 |
allow_self_hosted_omni = _bool_env(os.getenv("ALLOW_SELF_HOSTED_OMNI"), False)
|
| 115 |
mode_errors = _mode_consistency_errors(
|
|
@@ -139,11 +144,18 @@ class FigmentConfig:
|
|
| 139 |
nvidia_api_key=nvidia_api_key,
|
| 140 |
nvidia_base_url=os.getenv("NVIDIA_BASE_URL", NVIDIA_API_BASE_URL).strip() or NVIDIA_API_BASE_URL,
|
| 141 |
nvidia_model_id=os.getenv("NVIDIA_MODEL_ID", NVIDIA_OMNI_API_MODEL_ID).strip() or NVIDIA_OMNI_API_MODEL_ID,
|
| 142 |
-
local_model_id=os.getenv(
|
| 143 |
-
|
|
|
|
|
|
|
|
|
|
| 144 |
omni_endpoint_url=omni_endpoint_url,
|
| 145 |
hf_endpoint_url=hf_endpoint_url,
|
| 146 |
llama_base_url=os.getenv("LLAMA_BASE_URL", "http://127.0.0.1:8001/v1").strip(),
|
|
|
|
|
|
|
|
|
|
|
|
|
| 147 |
trace_dir=Path(os.getenv("FIGMENT_TRACE_DIR", "traces").strip() or "traces"),
|
| 148 |
).validated()
|
| 149 |
|
|
@@ -159,6 +171,8 @@ class FigmentConfig:
|
|
| 159 |
errors.append("MODEL_BACKEND=hosted_text_nemotron is retired; use MODEL_BACKEND=llama_cpp for local_4b_parakeet")
|
| 160 |
elif self.model_backend not in MODEL_BACKENDS:
|
| 161 |
errors.append(f"MODEL_BACKEND must be one of {sorted(MODEL_BACKENDS)}")
|
|
|
|
|
|
|
| 162 |
if self.audio_backend not in AUDIO_BACKENDS:
|
| 163 |
errors.append(f"AUDIO_BACKEND must be one of {sorted(AUDIO_BACKENDS)}")
|
| 164 |
if self.audio_backend == "parakeet_nemo" and not self.allow_local_asr:
|
|
@@ -188,7 +202,7 @@ class FigmentConfig:
|
|
| 188 |
def active_model_id(self) -> str:
|
| 189 |
if self.model_backend == "hosted_omni":
|
| 190 |
return self.nvidia_model_id
|
| 191 |
-
if self.model_backend
|
| 192 |
return self.local_model_id
|
| 193 |
return FIGMENT_CANNED_MODEL_ID
|
| 194 |
|
|
|
|
| 15 |
NVIDIA_OMNI_API_MODEL_ID = "nvidia/nemotron-3-nano-omni-30b-a3b-reasoning"
|
| 16 |
NVIDIA_API_BASE_URL = "https://integrate.api.nvidia.com/v1"
|
| 17 |
FIGMENT_CANNED_MODEL_ID = "figment-canned-deterministic"
|
| 18 |
+
FIGMENT_V14P_MODEL_REPO = "build-small-hackathon/figment-finetuned-model-archive"
|
| 19 |
+
FIGMENT_V14P_MODEL_SUBFOLDER = "figment_sft_v14p/figment-sft-v14p-lora-merged-bf16"
|
| 20 |
+
FIGMENT_V14P_MODEL_ID = "figment-sft-v14p-lora-merged-bf16"
|
| 21 |
|
| 22 |
MODEL_STACKS = {"omni_native", "local_4b_parakeet"}
|
| 23 |
+
MODEL_BACKENDS = {"hosted_omni", "llama_cpp", "hf_zerogpu", "canned"}
|
| 24 |
AUDIO_BACKENDS = {"omni_native", "parakeet_nemo", "canned", "none"}
|
| 25 |
FIGMENT_MODES = {"hosted", "local", "canned"}
|
| 26 |
|
|
|
|
| 89 |
omni_endpoint_url: str = ""
|
| 90 |
hf_endpoint_url: str = ""
|
| 91 |
llama_base_url: str = "http://127.0.0.1:8001/v1"
|
| 92 |
+
zerogpu_model_repo: str = FIGMENT_V14P_MODEL_REPO
|
| 93 |
+
zerogpu_model_subfolder: str = FIGMENT_V14P_MODEL_SUBFOLDER
|
| 94 |
trace_dir: Path = Path("traces")
|
| 95 |
|
| 96 |
@classmethod
|
|
|
|
| 114 |
backend = {"hosted": "hosted_omni", "local": "llama_cpp", "canned": "canned"}[backend]
|
| 115 |
stack = os.getenv("MODEL_STACK", "").strip()
|
| 116 |
if not stack:
|
| 117 |
+
stack = "local_4b_parakeet" if backend in {"llama_cpp", "hf_zerogpu"} else "omni_native"
|
| 118 |
audio = os.getenv("AUDIO_BACKEND", "none").strip() or "none"
|
| 119 |
allow_self_hosted_omni = _bool_env(os.getenv("ALLOW_SELF_HOSTED_OMNI"), False)
|
| 120 |
mode_errors = _mode_consistency_errors(
|
|
|
|
| 144 |
nvidia_api_key=nvidia_api_key,
|
| 145 |
nvidia_base_url=os.getenv("NVIDIA_BASE_URL", NVIDIA_API_BASE_URL).strip() or NVIDIA_API_BASE_URL,
|
| 146 |
nvidia_model_id=os.getenv("NVIDIA_MODEL_ID", NVIDIA_OMNI_API_MODEL_ID).strip() or NVIDIA_OMNI_API_MODEL_ID,
|
| 147 |
+
local_model_id=os.getenv(
|
| 148 |
+
"LOCAL_MODEL_ID",
|
| 149 |
+
FIGMENT_V14P_MODEL_ID if backend == "hf_zerogpu" else NVIDIA_NEMOTRON_3_NANO_4B_BF16_MODEL_ID,
|
| 150 |
+
).strip()
|
| 151 |
+
or (FIGMENT_V14P_MODEL_ID if backend == "hf_zerogpu" else NVIDIA_NEMOTRON_3_NANO_4B_BF16_MODEL_ID),
|
| 152 |
omni_endpoint_url=omni_endpoint_url,
|
| 153 |
hf_endpoint_url=hf_endpoint_url,
|
| 154 |
llama_base_url=os.getenv("LLAMA_BASE_URL", "http://127.0.0.1:8001/v1").strip(),
|
| 155 |
+
zerogpu_model_repo=os.getenv("ZEROGPU_MODEL_REPO", FIGMENT_V14P_MODEL_REPO).strip()
|
| 156 |
+
or FIGMENT_V14P_MODEL_REPO,
|
| 157 |
+
zerogpu_model_subfolder=os.getenv("ZEROGPU_MODEL_SUBFOLDER", FIGMENT_V14P_MODEL_SUBFOLDER).strip()
|
| 158 |
+
or FIGMENT_V14P_MODEL_SUBFOLDER,
|
| 159 |
trace_dir=Path(os.getenv("FIGMENT_TRACE_DIR", "traces").strip() or "traces"),
|
| 160 |
).validated()
|
| 161 |
|
|
|
|
| 171 |
errors.append("MODEL_BACKEND=hosted_text_nemotron is retired; use MODEL_BACKEND=llama_cpp for local_4b_parakeet")
|
| 172 |
elif self.model_backend not in MODEL_BACKENDS:
|
| 173 |
errors.append(f"MODEL_BACKEND must be one of {sorted(MODEL_BACKENDS)}")
|
| 174 |
+
if self.model_backend == "hf_zerogpu" and self.model_stack != "local_4b_parakeet":
|
| 175 |
+
errors.append("MODEL_BACKEND=hf_zerogpu requires MODEL_STACK=local_4b_parakeet")
|
| 176 |
if self.audio_backend not in AUDIO_BACKENDS:
|
| 177 |
errors.append(f"AUDIO_BACKEND must be one of {sorted(AUDIO_BACKENDS)}")
|
| 178 |
if self.audio_backend == "parakeet_nemo" and not self.allow_local_asr:
|
|
|
|
| 202 |
def active_model_id(self) -> str:
|
| 203 |
if self.model_backend == "hosted_omni":
|
| 204 |
return self.nvidia_model_id
|
| 205 |
+
if self.model_backend in {"llama_cpp", "hf_zerogpu"} or self.model_stack == "local_4b_parakeet":
|
| 206 |
return self.local_model_id
|
| 207 |
return FIGMENT_CANNED_MODEL_ID
|
| 208 |
|
figment/model_client.py
CHANGED
|
@@ -15,6 +15,7 @@ import wave
|
|
| 15 |
from typing import Any
|
| 16 |
|
| 17 |
from .config import FigmentConfig, load_config
|
|
|
|
| 18 |
|
| 19 |
|
| 20 |
class ModelClientError(RuntimeError):
|
|
@@ -126,6 +127,13 @@ class ModelClient:
|
|
| 126 |
model_id=self.config.local_model_id,
|
| 127 |
auth_headers={},
|
| 128 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 129 |
if self.config.model_backend == "hosted_omni":
|
| 130 |
endpoint = self.config.omni_endpoint_url or self.config.hf_endpoint_url or self.config.nvidia_base_url
|
| 131 |
if not endpoint:
|
|
|
|
| 15 |
from typing import Any
|
| 16 |
|
| 17 |
from .config import FigmentConfig, load_config
|
| 18 |
+
from .zerogpu_runtime import generate_zero_gpu_json
|
| 19 |
|
| 20 |
|
| 21 |
class ModelClientError(RuntimeError):
|
|
|
|
| 127 |
model_id=self.config.local_model_id,
|
| 128 |
auth_headers={},
|
| 129 |
)
|
| 130 |
+
if self.config.model_backend == "hf_zerogpu":
|
| 131 |
+
return generate_zero_gpu_json(
|
| 132 |
+
prompt=prompt,
|
| 133 |
+
model_repo=self.config.zerogpu_model_repo,
|
| 134 |
+
model_subfolder=self.config.zerogpu_model_subfolder,
|
| 135 |
+
model_id=self.config.local_model_id,
|
| 136 |
+
)
|
| 137 |
if self.config.model_backend == "hosted_omni":
|
| 138 |
endpoint = self.config.omni_endpoint_url or self.config.hf_endpoint_url or self.config.nvidia_base_url
|
| 139 |
if not endpoint:
|
figment/trace.py
CHANGED
|
@@ -135,7 +135,7 @@ def _derive_final_route(
|
|
| 135 |
return "model_repaired"
|
| 136 |
if fallback_tier == "canned":
|
| 137 |
return "canned_backend"
|
| 138 |
-
if raw_route in {"hosted_omni", "llama_cpp"}:
|
| 139 |
return "live_model_generated"
|
| 140 |
return raw_route or "unknown"
|
| 141 |
|
|
|
|
| 135 |
return "model_repaired"
|
| 136 |
if fallback_tier == "canned":
|
| 137 |
return "canned_backend"
|
| 138 |
+
if raw_route in {"hosted_omni", "llama_cpp", "hf_zerogpu"}:
|
| 139 |
return "live_model_generated"
|
| 140 |
return raw_route or "unknown"
|
| 141 |
|
figment/zerogpu_runtime.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""ZeroGPU Transformers runtime for Figment's published v14p model."""
|
| 2 |
+
|
| 3 |
+
from __future__ import annotations
|
| 4 |
+
|
| 5 |
+
import json
|
| 6 |
+
import os
|
| 7 |
+
import threading
|
| 8 |
+
from typing import Any
|
| 9 |
+
|
| 10 |
+
|
| 11 |
+
try: # pragma: no cover - exercised in Hugging Face Spaces.
|
| 12 |
+
import spaces
|
| 13 |
+
except ImportError: # pragma: no cover - local tests use the no-op decorator.
|
| 14 |
+
|
| 15 |
+
class _SpacesCompat:
|
| 16 |
+
@staticmethod
|
| 17 |
+
def GPU(*_: Any, **__: Any) -> Any: # noqa: N802 - mirrors the spaces API.
|
| 18 |
+
def decorator(function: Any) -> Any:
|
| 19 |
+
return function
|
| 20 |
+
|
| 21 |
+
return decorator
|
| 22 |
+
|
| 23 |
+
spaces = _SpacesCompat()
|
| 24 |
+
|
| 25 |
+
|
| 26 |
+
_RUNTIMES: dict[tuple[str, str], "_ZeroGpuRuntime"] = {}
|
| 27 |
+
_RUNTIME_LOCK = threading.Lock()
|
| 28 |
+
|
| 29 |
+
|
| 30 |
+
@spaces.GPU(
|
| 31 |
+
duration=int(os.getenv("ZEROGPU_DURATION_SECONDS", "120")),
|
| 32 |
+
size=os.getenv("ZEROGPU_SIZE", "large"),
|
| 33 |
+
)
|
| 34 |
+
def generate_zero_gpu_json(
|
| 35 |
+
*,
|
| 36 |
+
prompt: str,
|
| 37 |
+
model_repo: str,
|
| 38 |
+
model_subfolder: str,
|
| 39 |
+
model_id: str,
|
| 40 |
+
) -> dict[str, Any]:
|
| 41 |
+
runtime = _runtime_for(model_repo, model_subfolder, model_id)
|
| 42 |
+
return runtime.generate_json(prompt)
|
| 43 |
+
|
| 44 |
+
|
| 45 |
+
def _runtime_for(model_repo: str, model_subfolder: str, model_id: str) -> "_ZeroGpuRuntime":
|
| 46 |
+
key = (model_repo, model_subfolder)
|
| 47 |
+
with _RUNTIME_LOCK:
|
| 48 |
+
runtime = _RUNTIMES.get(key)
|
| 49 |
+
if runtime is None:
|
| 50 |
+
runtime = _ZeroGpuRuntime(model_repo=model_repo, model_subfolder=model_subfolder, model_id=model_id)
|
| 51 |
+
_RUNTIMES[key] = runtime
|
| 52 |
+
return runtime
|
| 53 |
+
|
| 54 |
+
|
| 55 |
+
class _ZeroGpuRuntime:
|
| 56 |
+
def __init__(self, *, model_repo: str, model_subfolder: str, model_id: str) -> None:
|
| 57 |
+
import torch
|
| 58 |
+
from transformers import AutoModelForCausalLM
|
| 59 |
+
from transformers import AutoTokenizer
|
| 60 |
+
|
| 61 |
+
self.model_id = model_id
|
| 62 |
+
self.max_context_tokens = int(os.getenv("ZEROGPU_MAX_CONTEXT_TOKENS", "16384"))
|
| 63 |
+
self.max_generation_tokens = int(os.getenv("ZEROGPU_MAX_GENERATION_TOKENS", "1536"))
|
| 64 |
+
self.lock = threading.Lock()
|
| 65 |
+
self.torch = torch
|
| 66 |
+
self.tokenizer = AutoTokenizer.from_pretrained(
|
| 67 |
+
model_repo,
|
| 68 |
+
subfolder=model_subfolder,
|
| 69 |
+
trust_remote_code=True,
|
| 70 |
+
)
|
| 71 |
+
if self.tokenizer.pad_token is None:
|
| 72 |
+
self.tokenizer.pad_token = self.tokenizer.eos_token
|
| 73 |
+
self.model = AutoModelForCausalLM.from_pretrained(
|
| 74 |
+
model_repo,
|
| 75 |
+
subfolder=model_subfolder,
|
| 76 |
+
trust_remote_code=True,
|
| 77 |
+
torch_dtype=torch.bfloat16,
|
| 78 |
+
)
|
| 79 |
+
self.model.to("cuda")
|
| 80 |
+
self.model.eval()
|
| 81 |
+
self.model.config.use_cache = True
|
| 82 |
+
|
| 83 |
+
def generate_json(self, prompt: str) -> dict[str, Any]:
|
| 84 |
+
messages = [{"role": "user", "content": prompt}]
|
| 85 |
+
with self.lock:
|
| 86 |
+
input_ids = self.tokenizer.apply_chat_template(
|
| 87 |
+
messages,
|
| 88 |
+
add_generation_prompt=True,
|
| 89 |
+
return_tensors="pt",
|
| 90 |
+
)
|
| 91 |
+
device = next(self.model.parameters()).device
|
| 92 |
+
input_ids = input_ids.to(device)
|
| 93 |
+
input_len = int(input_ids.shape[-1])
|
| 94 |
+
available_tokens = max(1, self.max_context_tokens - input_len - 8)
|
| 95 |
+
max_new_tokens = max(1, min(self.max_generation_tokens, available_tokens))
|
| 96 |
+
attention_mask = self.torch.ones_like(input_ids)
|
| 97 |
+
with self.torch.inference_mode():
|
| 98 |
+
output_ids = self.model.generate(
|
| 99 |
+
input_ids=input_ids,
|
| 100 |
+
attention_mask=attention_mask,
|
| 101 |
+
max_new_tokens=max_new_tokens,
|
| 102 |
+
do_sample=False,
|
| 103 |
+
pad_token_id=self.tokenizer.pad_token_id,
|
| 104 |
+
eos_token_id=self.tokenizer.eos_token_id,
|
| 105 |
+
)
|
| 106 |
+
generated_ids = output_ids[0, input_len:]
|
| 107 |
+
content = self.tokenizer.decode(generated_ids, skip_special_tokens=True).strip()
|
| 108 |
+
return _parse_json_object(content)
|
| 109 |
+
|
| 110 |
+
|
| 111 |
+
def _parse_json_object(content: str) -> dict[str, Any]:
|
| 112 |
+
text = content.strip()
|
| 113 |
+
try:
|
| 114 |
+
parsed = json.loads(text)
|
| 115 |
+
except json.JSONDecodeError:
|
| 116 |
+
decoder = json.JSONDecoder()
|
| 117 |
+
for index, char in enumerate(text):
|
| 118 |
+
if char != "{":
|
| 119 |
+
continue
|
| 120 |
+
try:
|
| 121 |
+
parsed, _ = decoder.raw_decode(text[index:])
|
| 122 |
+
break
|
| 123 |
+
except json.JSONDecodeError:
|
| 124 |
+
continue
|
| 125 |
+
else:
|
| 126 |
+
raise
|
| 127 |
+
if isinstance(parsed, dict):
|
| 128 |
+
return parsed
|
| 129 |
+
raise json.JSONDecodeError("model response JSON was not an object", text, 0)
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
def _preload_default_runtime_from_env() -> None:
|
| 133 |
+
if os.getenv("MODEL_BACKEND") != "hf_zerogpu":
|
| 134 |
+
return
|
| 135 |
+
if os.getenv("FIGMENT_SKIP_ZEROGPU_PRELOAD", "").strip().lower() in {"1", "true", "yes", "on"}:
|
| 136 |
+
return
|
| 137 |
+
model_repo = os.getenv("ZEROGPU_MODEL_REPO", "build-small-hackathon/figment-finetuned-model-archive").strip()
|
| 138 |
+
model_subfolder = os.getenv(
|
| 139 |
+
"ZEROGPU_MODEL_SUBFOLDER",
|
| 140 |
+
"figment_sft_v14p/figment-sft-v14p-lora-merged-bf16",
|
| 141 |
+
).strip()
|
| 142 |
+
model_id = os.getenv("LOCAL_MODEL_ID", "figment-sft-v14p-lora-merged-bf16").strip()
|
| 143 |
+
if model_repo and model_subfolder and model_id:
|
| 144 |
+
_runtime_for(model_repo, model_subfolder, model_id)
|
| 145 |
+
|
| 146 |
+
|
| 147 |
+
_preload_default_runtime_from_env()
|
requirements.txt
CHANGED
|
@@ -5,3 +5,15 @@ httpx>=0.27,<1
|
|
| 5 |
pydantic>=2.7,<3
|
| 6 |
python-dotenv>=1.0,<2
|
| 7 |
PyYAML>=6.0,<7
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 5 |
pydantic>=2.7,<3
|
| 6 |
python-dotenv>=1.0,<2
|
| 7 |
PyYAML>=6.0,<7
|
| 8 |
+
spaces>=0.40,<1
|
| 9 |
+
torch>=2.8,<2.12
|
| 10 |
+
transformers>=4.52,<5
|
| 11 |
+
accelerate>=1.8,<2
|
| 12 |
+
einops>=0.8,<1
|
| 13 |
+
protobuf>=5,<7
|
| 14 |
+
safetensors>=0.5,<1
|
| 15 |
+
sentencepiece>=0.2,<1
|
| 16 |
+
setuptools>=70
|
| 17 |
+
wheel>=0.45
|
| 18 |
+
causal-conv1d>=1.6,<2
|
| 19 |
+
mamba-ssm>=2.2,<3
|
tests/test_model_client.py
CHANGED
|
@@ -145,6 +145,38 @@ def test_local_llama_cpp_uses_local_base_url_without_hosted_auth(monkeypatch: An
|
|
| 145 |
assert captured["body"]["model"] == "local-test-model"
|
| 146 |
|
| 147 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 148 |
def test_custom_hf_endpoint_prefers_hf_token_over_nvidia_key(monkeypatch: Any) -> None:
|
| 149 |
captured: dict[str, Any] = {}
|
| 150 |
|
|
|
|
| 145 |
assert captured["body"]["model"] == "local-test-model"
|
| 146 |
|
| 147 |
|
| 148 |
+
def test_hf_zerogpu_backend_uses_published_v14p_runtime(monkeypatch: Any) -> None:
|
| 149 |
+
captured: dict[str, Any] = {}
|
| 150 |
+
|
| 151 |
+
def fake_generate_zero_gpu_json(*, prompt: str, model_repo: str, model_subfolder: str, model_id: str) -> dict[str, Any]:
|
| 152 |
+
captured["prompt"] = prompt
|
| 153 |
+
captured["model_repo"] = model_repo
|
| 154 |
+
captured["model_subfolder"] = model_subfolder
|
| 155 |
+
captured["model_id"] = model_id
|
| 156 |
+
return {"protocol_urgency": "monitor", "source_cards": []}
|
| 157 |
+
|
| 158 |
+
monkeypatch.setattr("figment.model_client.generate_zero_gpu_json", fake_generate_zero_gpu_json)
|
| 159 |
+
|
| 160 |
+
client = ModelClient(
|
| 161 |
+
FigmentConfig(
|
| 162 |
+
model_backend="hf_zerogpu",
|
| 163 |
+
model_stack="local_4b_parakeet",
|
| 164 |
+
local_model_id="figment-sft-v14p-lora-merged-bf16",
|
| 165 |
+
zerogpu_model_repo="build-small-hackathon/figment-finetuned-model-archive",
|
| 166 |
+
zerogpu_model_subfolder="figment_sft_v14p/figment-sft-v14p-lora-merged-bf16",
|
| 167 |
+
)
|
| 168 |
+
)
|
| 169 |
+
result = client.generate_json("Return JSON.", {})
|
| 170 |
+
|
| 171 |
+
assert result["protocol_urgency"] == "monitor"
|
| 172 |
+
assert captured == {
|
| 173 |
+
"prompt": "Return JSON.",
|
| 174 |
+
"model_repo": "build-small-hackathon/figment-finetuned-model-archive",
|
| 175 |
+
"model_subfolder": "figment_sft_v14p/figment-sft-v14p-lora-merged-bf16",
|
| 176 |
+
"model_id": "figment-sft-v14p-lora-merged-bf16",
|
| 177 |
+
}
|
| 178 |
+
|
| 179 |
+
|
| 180 |
def test_custom_hf_endpoint_prefers_hf_token_over_nvidia_key(monkeypatch: Any) -> None:
|
| 181 |
captured: dict[str, Any] = {}
|
| 182 |
|
tests/test_runtime_honesty.py
CHANGED
|
@@ -12,10 +12,12 @@ CONFIG_ENV_KEYS = (
|
|
| 12 |
"MODEL_STACK",
|
| 13 |
"MODEL_BACKEND",
|
| 14 |
"AUDIO_BACKEND",
|
| 15 |
-
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
|
|
|
|
|
|
|
| 19 |
"HF_MODEL_ID",
|
| 20 |
"NVIDIA_MODEL_ID",
|
| 21 |
"NVIDIA_BASE_URL",
|
|
@@ -73,10 +75,27 @@ def test_env_config_allows_explicit_local_self_hosted_omni(monkeypatch: pytest.M
|
|
| 73 |
assert config.active_model_id == "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16"
|
| 74 |
|
| 75 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 76 |
@pytest.mark.parametrize(
|
| 77 |
("model_route", "events", "expected_final_route"),
|
| 78 |
[
|
| 79 |
({"model_backend": "hosted_omni", "fallback_reason": None}, [], "live_model_generated"),
|
|
|
|
| 80 |
({"model_backend": "hosted_omni", "fallback_reason": None}, ["navigator output repaired by hosted retry"], "model_repaired"),
|
| 81 |
(
|
| 82 |
{"model_backend": "hosted_omni", "fallback_reason": None, "field_level_fallback_used": True},
|
|
|
|
| 12 |
"MODEL_STACK",
|
| 13 |
"MODEL_BACKEND",
|
| 14 |
"AUDIO_BACKEND",
|
| 15 |
+
"ENABLE_AUDIO_INTAKE",
|
| 16 |
+
"ALLOW_LOCAL_ASR",
|
| 17 |
+
"ALLOW_SELF_HOSTED_OMNI",
|
| 18 |
+
"ALLOW_STRETCH_STACK",
|
| 19 |
+
"ZEROGPU_MODEL_REPO",
|
| 20 |
+
"ZEROGPU_MODEL_SUBFOLDER",
|
| 21 |
"HF_MODEL_ID",
|
| 22 |
"NVIDIA_MODEL_ID",
|
| 23 |
"NVIDIA_BASE_URL",
|
|
|
|
| 75 |
assert config.active_model_id == "nvidia/Nemotron-3-Nano-Omni-30B-A3B-Reasoning-BF16"
|
| 76 |
|
| 77 |
|
| 78 |
+
def test_env_config_zerogpu_backend_defaults_to_v14p(monkeypatch: pytest.MonkeyPatch, tmp_path) -> None:
|
| 79 |
+
monkeypatch.chdir(tmp_path)
|
| 80 |
+
_clear_config_env(monkeypatch)
|
| 81 |
+
monkeypatch.setenv("FIGMENT_MODE", "hosted")
|
| 82 |
+
monkeypatch.setenv("MODEL_BACKEND", "hf_zerogpu")
|
| 83 |
+
|
| 84 |
+
config = FigmentConfig.from_env()
|
| 85 |
+
|
| 86 |
+
assert config.model_backend == "hf_zerogpu"
|
| 87 |
+
assert config.model_stack == "local_4b_parakeet"
|
| 88 |
+
assert config.local_model_id == "figment-sft-v14p-lora-merged-bf16"
|
| 89 |
+
assert config.zerogpu_model_repo == "build-small-hackathon/figment-finetuned-model-archive"
|
| 90 |
+
assert config.zerogpu_model_subfolder == "figment_sft_v14p/figment-sft-v14p-lora-merged-bf16"
|
| 91 |
+
assert config.active_model_id == "figment-sft-v14p-lora-merged-bf16"
|
| 92 |
+
|
| 93 |
+
|
| 94 |
@pytest.mark.parametrize(
|
| 95 |
("model_route", "events", "expected_final_route"),
|
| 96 |
[
|
| 97 |
({"model_backend": "hosted_omni", "fallback_reason": None}, [], "live_model_generated"),
|
| 98 |
+
({"model_backend": "hf_zerogpu", "fallback_reason": None}, [], "live_model_generated"),
|
| 99 |
({"model_backend": "hosted_omni", "fallback_reason": None}, ["navigator output repaired by hosted retry"], "model_repaired"),
|
| 100 |
(
|
| 101 |
{"model_backend": "hosted_omni", "fallback_reason": None, "field_level_fallback_used": True},
|