Add image-to-video and automatic 15-second duration

Adds native LTX-2.5 image conditioning, model-selected duration capped at 15 seconds, resilient optional pipelines, updated controls, and validation documentation.

README.md

@@ -15,7 +15,7 @@ models:
 
 # LTX-2.5 Video + Audio
 
-An interactive **text-to-video with synchronized audio** demo for
+An interactive **text/image-to-video with synchronized audio** demo for
 [Lightricks LTX-2.5](https://huggingface.co/Lightricks/LTX-2.5), using the
 official [Diffusers-compatible checkpoint](https://huggingface.co/Lightricks/LTX-2.5-Diffusers).
 
@@ -25,12 +25,12 @@ It runs the model's published distilled two-stage recipe:
 2. ×2 spatial latent upsampling.
 3. 3 steps at final resolution, including audio-latent refinement.
 
-The app is intended for a **ZeroGPU xlarge** worker. It offers known-good
-landscape, portrait, square, and cinema-resolution presets up to 1536 pixels per
-side; manual clips up to 8 seconds; an optional remote LTX prompt enhancer; and
-both the tiled convolutional decoder and LTX-2.5's detail-focused diffusion
-decoder. High-resolution and long-duration jobs consume more shared ZeroGPU
-quota and may wait longer in the queue.
+The app is intended for a **ZeroGPU xlarge** worker. It offers text-to-video and
+image-to-video; landscape, portrait, square, and cinema-resolution presets up to
+1536 pixels per side; manual clips up to 15 seconds; LTX-2.5 automatic duration;
+an optional remote LTX prompt enhancer; and both the tiled convolutional decoder
+and LTX-2.5's detail-focused diffusion decoder. High-resolution and long-duration
+jobs consume more shared ZeroGPU quota and may wait longer in the queue.
 
 ## Required Space secret
 
@@ -44,6 +44,8 @@ repository.
 
 - Width and height must each be divisible by 64 for the two-stage pipeline.
 - Clip length is aligned to the required `8k + 1` frame grid.
+- Automatic duration reserves ZeroGPU for the 15-second ceiling even when the
+  model chooses a shorter clip.
 - The public API intentionally separates `/prepare_prompt` (CPU/remote prompt
   enhancement) from `/generate_video` (GPU rendering), so prompt enhancement
   never wastes a GPU reservation.

app.py

@@ -7,7 +7,6 @@ latent x2 spatial upsampling, then 3 final-resolution denoising steps.
 
 import os
 import tempfile
-from functools import lru_cache
 
 # Configure the allocator before Torch initializes. This avoids a known
 # allocator assertion on the ZeroGPU MIG runtime.
@@ -17,9 +16,11 @@ os.environ.setdefault("PYTORCH_CUDA_ALLOC_CONF", "backend:cudaMallocAsync")
 # `spaces` must load before Torch so ZeroGPU can intercept CUDA placement.
 import spaces
 import gradio as gr
+import PIL.Image
 import torch
-from gradio_client import Client
+from gradio_client import Client, handle_file
 from diffusers import (
+    LTX2ImageToVideoPipeline,
     LTX2LatentUpsamplePipeline,
     LTX2Pipeline,
     LTX2VideoDiffusionDecodePipeline,
@@ -42,7 +43,8 @@ MODEL_ID = "Lightricks/LTX-2.5-Diffusers"
 FRAME_RATE = 24.0
 MAX_SEED = 2**31 - 1
 MAX_SIDE = 1536
-MAX_DURATION_SECONDS = 8.0
+MAX_DURATION_SECONDS = 15.0
+AUTO_MAX_FRAMES = int(MAX_DURATION_SECONDS * FRAME_RATE) // 8 * 8 + 1
 ENHANCER_SPACE = "diffusers-internal-dev/LTX-2.4-Prompt-Enhancer"
 
 RESOLUTION_PRESETS = {
@@ -61,11 +63,21 @@ def frames_for_duration(seconds: float) -> int:
 
 
 def requested_gpu_seconds(
-    prompt, width, height, duration_seconds, seed, randomize_seed, decoder, *args, **kwargs
+    prompt,
+    image_path,
+    width,
+    height,
+    duration_seconds,
+    auto_length,
+    seed,
+    randomize_seed,
+    decoder,
+    *args,
+    **kwargs,
 ):
     """Reserve a realistic ZeroGPU duration based on output pixels and frames."""
-    del prompt, seed, randomize_seed, args, kwargs
-    frames = frames_for_duration(float(duration_seconds))
+    del prompt, image_path, seed, randomize_seed, args, kwargs
+    frames = AUTO_MAX_FRAMES if auto_length else frames_for_duration(float(duration_seconds))
     pixels = int(width) * int(height) * frames
     # Calibrated against this Space's live calls: 1024x576 at 2s/4s took
     # 15.1s/18.8s and 1472x832 at 2s took 21.1s, including public API overhead.
@@ -77,10 +89,12 @@ def requested_gpu_seconds(
 
 HF_TOKEN = os.environ.get("HF_TOKEN")
 pipeline = None
+image_pipeline = None
 upsample_pipeline = None
 diffusion_decode_pipeline = None
 AUDIO_SAMPLE_RATE = None
 MODEL_LOAD_ERROR = None
+IMAGE_PIPELINE_ERROR = None
 DIFFUSION_DECODER_ERROR = None
 
 # A new Space can safely boot before its owner adds the gated-model secret. It
@@ -102,6 +116,7 @@ else:
                 "model_index.json", "audio_vae/*", "connectors/*", "scheduler/*",
                 "text_encoder/*", "tokenizer/*", "transformer/*", "vae/*",
                 "vocoder/*", "latent_upsampler/*", "diffusion_decoder/*",
+                "duration_head/*",
             ],
             ignore_patterns=[
                 "transformer/*-of-00008.safetensors",
@@ -116,7 +131,7 @@ else:
         # the GPU worker when a decorated handler runs.
         pipeline = LTX2Pipeline.from_pretrained(
             model_dir, dtype=torch.bfloat16,
-            prompt_enhancer=None, processor=None, duration_head=None,
+            prompt_enhancer=None, processor=None,
         )
         latent_upsampler = LTX2LatentUpsamplerModel.from_pretrained(
             model_dir,
@@ -130,6 +145,27 @@ else:
             vae=pipeline.vae,
             latent_upsampler=latent_upsampler,
         )
+        try:
+            image_pipeline = LTX2ImageToVideoPipeline(
+                scheduler=pipeline.scheduler,
+                vae=pipeline.vae,
+                audio_vae=pipeline.audio_vae,
+                text_encoder=pipeline.text_encoder,
+                tokenizer=pipeline.tokenizer,
+                connectors=pipeline.connectors,
+                transformer=pipeline.transformer,
+                vocoder=pipeline.vocoder,
+                processor=getattr(pipeline, "processor", None),
+                prompt_enhancer=getattr(pipeline, "prompt_enhancer", None),
+                duration_head=getattr(pipeline, "duration_head", None),
+            )
+        except Exception as image_error:  # noqa: BLE001 - text generation remains usable.
+            IMAGE_PIPELINE_ERROR = type(image_error).__name__
+            print(
+                f"[ltx-2-5] Image-to-video unavailable ({image_error!r}); "
+                "text-to-video remains enabled.",
+                flush=True,
+            )
         try:
             diffusion_decoder = LTX2VideoDiffusionDecoderModel.from_pretrained(
                 model_dir,
@@ -153,7 +189,13 @@ else:
                 flush=True,
             )
         AUDIO_SAMPLE_RATE = pipeline.vocoder.config.output_sampling_rate
-        print("[ltx-2-5] Model is ready for ZeroGPU requests.", flush=True)
+        duration_status = "enabled" if getattr(pipeline, "duration_head", None) else "missing"
+        image_status = "enabled" if image_pipeline is not None else "unavailable"
+        print(
+            "[ltx-2-5] Model is ready for requests; "
+            f"image-to-video is {image_status}; automatic duration is {duration_status}.",
+            flush=True,
+        )
     except Exception as error:  # noqa: BLE001 - preserve a bootable setup screen.
         import traceback
         import transformers
@@ -163,14 +205,14 @@ else:
         print(f"[ltx-2-5] {MODEL_LOAD_ERROR}", flush=True)
 
 
-@lru_cache(maxsize=64)
-def enhance_prompt_cached(prompt: str) -> str:
-    """Expand a short prompt using the official remote LTX prompt enhancer."""
+def enhance_prompt_remote(prompt: str, image_path: str | None) -> str:
+    """Expand text/image intent using the official remote LTX prompt enhancer."""
     client = Client(ENHANCER_SPACE, token=HF_TOKEN, httpx_kwargs={"timeout": 300.0})
-    return client.predict(prompt, None, api_name="/enhance")
+    image = handle_file(image_path) if image_path else None
+    return client.predict(prompt, image, api_name="/enhance")
 
 
-def prepare_prompt(prompt: str, enhance: bool) -> str:
+def prepare_prompt(prompt: str, image_path: str | None, enhance: bool) -> str:
     """Validate and optionally enrich a prompt before reserving the GPU worker."""
     cleaned = (prompt or "").strip()
     if not cleaned:
@@ -178,7 +220,7 @@ def prepare_prompt(prompt: str, enhance: bool) -> str:
     if not enhance:
         return cleaned
     try:
-        return enhance_prompt_cached(cleaned) or cleaned
+        return enhance_prompt_remote(cleaned, image_path) or cleaned
     except Exception as error:  # noqa: BLE001 - generation can safely use the raw prompt.
         print(f"[ltx-2-5] Prompt enhancement failed ({error!r}).", flush=True)
         gr.Warning(
@@ -188,7 +230,7 @@ def prepare_prompt(prompt: str, enhance: bool) -> str:
         return cleaned
 
 
-def apply_resolution_preset(preset: str) -> tuple[gr.Slider, gr.Slider]:
+def apply_resolution_preset(preset: str) -> tuple[object, object]:
     """Apply a known-good LTX-2.5 resolution while keeping manual controls available."""
     dimensions = RESOLUTION_PRESETS.get(preset)
     if dimensions is None:
@@ -196,17 +238,57 @@ def apply_resolution_preset(preset: str) -> tuple[gr.Slider, gr.Slider]:
     return dimensions
 
 
+def load_conditioning_image(
+    path: str, width: int, height: int
+) -> PIL.Image.Image:
+    """Cover-resize and center-crop an image without distorting its aspect ratio."""
+    with PIL.Image.open(path) as source:
+        image = source.convert("RGB")
+    scale = max(width / image.width, height / image.height)
+    image = image.resize(
+        (round(image.width * scale), round(image.height * scale)),
+        PIL.Image.Resampling.LANCZOS,
+    )
+    left = (image.width - width) // 2
+    top = (image.height - height) // 2
+    return image.crop((left, top, left + width, top + height))
+
+
+def dimensions_for_image(path: str | None) -> tuple[object, object]:
+    """Choose a /64 cinema shape that preserves an uploaded image's orientation."""
+    if not path:
+        return gr.update(), gr.update()
+    with PIL.Image.open(path) as image:
+        image_width, image_height = image.size
+    short_side = 832
+    if image_width >= image_height:
+        height = short_side
+        width = min(
+            MAX_SIDE,
+            max(short_side, round(short_side * image_width / image_height / 64) * 64),
+        )
+    else:
+        width = short_side
+        height = min(
+            MAX_SIDE,
+            max(short_side, round(short_side * image_height / image_width / 64) * 64),
+        )
+    return width, height
+
+
 @spaces.GPU(duration=requested_gpu_seconds, size="xlarge")
 def generate(
     prompt: str,
+    image_path: str | None,
     width: int,
     height: int,
     duration_seconds: float,
+    auto_length: bool,
     seed: int,
     randomize_seed: bool,
     decoder: str,
 ) -> tuple[str, int, str]:
-    """Generate an LTX-2.5 video with synchronized audio from a text prompt."""
+    """Generate an LTX-2.5 video with synchronized audio from text or an image."""
     if pipeline is None or upsample_pipeline is None or AUDIO_SAMPLE_RATE is None:
         access_help = (
             "This Space needs an HF_TOKEN Secret for the gated LTX-2.5 checkpoint. "
@@ -219,14 +301,23 @@ def generate(
         )
     if not prompt or not prompt.strip():
         raise gr.Error("Enter a prompt before generating.")
+    if image_path and image_pipeline is None:
+        detail = f" ({IMAGE_PIPELINE_ERROR})" if IMAGE_PIPELINE_ERROR else ""
+        raise gr.Error(
+            f"Image-to-video is unavailable{detail}; remove the image and retry."
+        )
 
     width, height = int(width), int(height)
     if width % 64 or height % 64:
         raise gr.Error("Width and height must both be divisible by 64 for two-stage LTX-2.5.")
     if not (512 <= width <= MAX_SIDE and 512 <= height <= MAX_SIDE):
         raise gr.Error(f"Width and height must be between 512 and {MAX_SIDE} pixels.")
-    if not (1.0 <= float(duration_seconds) <= MAX_DURATION_SECONDS):
+    if not auto_length and not (
+        1.0 <= float(duration_seconds) <= MAX_DURATION_SECONDS
+    ):
         raise gr.Error(f"Duration must be between 1 and {MAX_DURATION_SECONDS:g} seconds.")
+    if decoder not in {"conv", "diffusion"}:
+        raise gr.Error("Decoder must be either conv or diffusion.")
     if decoder == "diffusion" and diffusion_decode_pipeline is None:
         detail = f" ({DIFFUSION_DECODER_ERROR})" if DIFFUSION_DECODER_ERROR else ""
         raise gr.Error(f"The diffusion decoder is unavailable{detail}; choose conv and retry.")
@@ -234,8 +325,9 @@ def generate(
     if randomize_seed:
         seed = torch.randint(0, MAX_SEED, (1,), device="cuda").item()
     seed = int(seed)
-    num_frames = frames_for_duration(float(duration_seconds))
+    num_frames = None if auto_length else frames_for_duration(float(duration_seconds))
     generator = torch.Generator("cuda").manual_seed(seed)
+    active_pipeline = image_pipeline if image_path else pipeline
 
     shared = {
         "prompt": prompt.strip(),
@@ -255,15 +347,25 @@ def generate(
         "generator": generator,
         "return_dict": False,
     }
+    if image_path:
+        shared["image"] = load_conditioning_image(image_path, width, height)
 
     # Stage 1: half-resolution latent generation with the 8-step schedule.
-    stage_1_latents, stage_1_audio_latents = pipeline(
-        height=height // 2,
-        width=width // 2,
-        num_frames=num_frames,
-        sigmas=DISTILLED_SIGMA_VALUES,
-        output_type="latent",
+    stage_1_args = {
+        "height": height // 2,
+        "width": width // 2,
+        "num_frames": num_frames,
+        "sigmas": DISTILLED_SIGMA_VALUES,
+        "output_type": "latent",
         **shared,
+    }
+    if auto_length:
+        stage_1_args["max_seconds"] = MAX_DURATION_SECONDS
+    stage_1_latents, stage_1_audio_latents = active_pipeline(
+        **stage_1_args,
+    )
+    num_frames = (
+        (stage_1_latents.shape[2] - 1) * pipeline.vae_temporal_compression_ratio + 1
     )
 
     # The official recipe uses an x2 spatial latent upsampler between stages.
@@ -276,7 +378,7 @@ def generate(
     # Stage 2: continue the same random-noise stream at full resolution. The
     # audio latents are refined too, preserving synchronized audio output.
     use_diffusion_decoder = decoder == "diffusion"
-    stage_2_output = pipeline(
+    stage_2_output = active_pipeline(
         num_frames=num_frames,
         sigmas=STAGE_2_DISTILLED_SIGMA_VALUES,
         latents=upsampled_latents,
@@ -329,11 +431,13 @@ Diffusers-compatible weights. It renders a video and synchronized audio in two
 stages: 8 denoising steps at half resolution, latent upsampling, then 3 final
 steps at full resolution.
 
-The default cinema preset renders 1472×832. Clips can run up to 8 seconds, and
-the prompt enhancer turns short ideas into shot-level descriptions before GPU
-time starts. Use the convolutional decoder for high resolution and the optional
-diffusion decoder when you want a slower detail-focused finish. Outputs are
-probabilistic; reuse a seed to reproduce a composition.
+The default cinema preset renders 1472×832. Text-to-video and image-to-video
+clips can run up to 15 seconds, or LTX-2.5 can choose the duration from the
+prompt. The prompt enhancer turns short ideas or an uploaded reference image
+into shot-level descriptions before GPU time starts. Use the convolutional
+decoder for high resolution and the optional diffusion decoder when you want a
+slower detail-focused finish. Outputs are probabilistic; reuse a seed to
+reproduce a composition.
 
 [Model card](https://huggingface.co/Lightricks/LTX-2.5) ·
 [Diffusers checkpoint](https://huggingface.co/Lightricks/LTX-2.5-Diffusers) ·
@@ -377,6 +481,10 @@ with gr.Blocks(title="LTX-2.5 Video + Audio", fill_width=True) as demo:
     gr.Markdown(SETUP_NOTICE)
     with gr.Row():
         with gr.Column(scale=1):
+            image_input = gr.Image(
+                label="Starting image (optional · image-to-video)",
+                type="filepath",
+            )
             prompt_input = gr.Textbox(
                 label="Prompt",
                 lines=5,
@@ -399,6 +507,14 @@ with gr.Blocks(title="LTX-2.5 Video + Audio", fill_width=True) as demo:
                     label="Duration (seconds at 24 fps)",
                     info="Snapped to LTX's required 8k + 1 frame grid.",
                 )
+                auto_length_input = gr.Checkbox(
+                    value=False,
+                    label="Automatic duration (up to 15 seconds)",
+                    info=(
+                        "Lets LTX-2.5 choose the clip length from the prompt. "
+                        "This reserves for the 15-second worst case on ZeroGPU."
+                    ),
+                )
                 decoder_input = gr.Radio(
                     choices=["conv", "diffusion"],
                     value="conv",
@@ -433,18 +549,26 @@ with gr.Blocks(title="LTX-2.5 Video + Audio", fill_width=True) as demo:
         outputs=[width_input, height_input],
         api_name=False,
     )
+    image_input.upload(
+        fn=dimensions_for_image,
+        inputs=image_input,
+        outputs=[width_input, height_input],
+        api_name=False,
+    )
     generation_inputs = [
         prompt_used,
+        image_input,
         width_input,
         height_input,
         duration_input,
+        auto_length_input,
         seed_input,
         randomize_input,
         decoder_input,
     ]
     prepare_event = generate_button.click(
         fn=prepare_prompt,
-        inputs=[prompt_input, enhance_input],
+        inputs=[prompt_input, image_input, enhance_input],
         outputs=prompt_used,
         api_name="prepare_prompt",
     )

requirements.txt

@@ -14,6 +14,7 @@ protobuf
 av
 imageio
 imageio-ffmpeg
+Pillow
 numpy
 scipy
 kernels