nishitanand commited on
Commit
18b215f
·
verified ·
1 Parent(s): 978bee6

Upload folder using huggingface_hub

Browse files
Files changed (5) hide show
  1. README.md +99 -0
  2. figma.ckpt +3 -0
  3. figma_model.py +91 -0
  4. inference.py +92 -0
  5. requirements.txt +7 -0
README.md ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ ---
2
+ license: cc-by-nc-4.0
3
+ language:
4
+ - en
5
+ tags:
6
+ - music
7
+ - music-information-retrieval
8
+ - music-retrieval
9
+ - audio-text-retrieval
10
+ - contrastive-learning
11
+ - clap
12
+ - muq
13
+ pipeline_tag: feature-extraction
14
+ library_name: pytorch
15
+ ---
16
+
17
+ # FIGMA: Towards FIne-Grained Music retrievAl (ACL 2026)
18
+
19
+ **Accepted to ACL 2026.**
20
+
21
+ FIGMA retrieves music from natural-language descriptions that specify **fine-grained
22
+ musical attributes** — tempo, key, chord progression, beat/time signature — in
23
+ addition to high-level attributes like genre and mood. This repository hosts the
24
+ trained model checkpoint and the code to run inference.
25
+
26
+ - **Paper:** https://arxiv.org/abs/2606.06615
27
+ - **Project page:** https://nishitanand.github.io/figma-website
28
+ - **Code:** https://github.com/nishitanand/FIGMA
29
+ - **Benchmark:** https://huggingface.co/datasets/nishitanand/FGMCaps-benchmark
30
+ - **Data:** https://huggingface.co/datasets/nishitanand/FGMCaps-data
31
+
32
+ ## Installation
33
+
34
+ ```bash
35
+ pip install torch transformers librosa soundfile numpy
36
+ pip3 install muq
37
+ ```
38
+
39
+ ## Download the checkpoint
40
+
41
+ ```python
42
+ from huggingface_hub import hf_hub_download
43
+ ckpt = hf_hub_download(repo_id="nishitanand/FIGMA", filename="figma.ckpt")
44
+ ```
45
+
46
+ ## Inference
47
+
48
+ ```python
49
+ import torch, librosa
50
+ from figma_model import Figma, get_tokenizer
51
+
52
+ device = "cuda"
53
+ model = Figma.from_checkpoint(ckpt, device=device) # ckpt from the download step
54
+ tok = get_tokenizer()
55
+
56
+ # Text embedding
57
+ t = tok(["a song in F minor at 120 BPM in 4/4 time"], return_tensors="pt",
58
+ padding="max_length", truncation=True, max_length=128).to(device)
59
+ text_emb = model.encode_text(t) # [1, 512], L2-normalized
60
+
61
+ # Audio embedding (24 kHz mono -> [B, 1, samples])
62
+ wav, _ = librosa.load("clip.wav", sr=24000)
63
+ wav = torch.tensor(wav)[None, None].to(device)
64
+ audio_emb = model.encode_audio(wav) # [1, 512], L2-normalized
65
+
66
+ similarity = (audio_emb @ text_emb.T).item()
67
+ ```
68
+
69
+ Command-line text→audio retrieval over a folder:
70
+
71
+ ```bash
72
+ python inference.py --checkpoint figma.ckpt \
73
+ --audio_dir ./clips --query "a jazzy track in C major at 90 bpm" --topk 5
74
+ ```
75
+
76
+ ## Intended use & limitations
77
+
78
+ - Intended for **non-commercial research** on music retrieval / music–text alignment.
79
+ - English-centric captions; fine-grained attributes cover tempo, key, chords, and
80
+ beat/time signature.
81
+
82
+ ## Citation
83
+
84
+ ```bibtex
85
+ @inproceedings{figma2026,
86
+ title = {FIGMA: Towards FIne-Grained Music retrievAl},
87
+ author = {Anand, Nishit and Seth, Ashish and Ghosh, Sreyan and Manocha, Dinesh and Duraiswami, Ramani},
88
+ booktitle = {Proceedings of the 64th Annual Meeting of the Association for Computational Linguistics},
89
+ year = {2026},
90
+ url = {https://arxiv.org/abs/2606.06615}
91
+ }
92
+ ```
93
+
94
+ ## License
95
+
96
+ Released for **non-commercial research use** under **CC BY-NC 4.0**. The checkpoint
97
+ bundles the MuQ encoder (weights CC BY-NC 4.0); the E5 encoder is MIT. You must
98
+ attribute MuQ and E5 and comply with CC BY-NC 4.0. The inference code is MIT-licensed
99
+ in the code repository.
figma.ckpt ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:91dd36da360405bcad6773e4e4fccf5e0e25b8565b2a35010c6e7839c4d40689
3
+ size 3838497572
figma_model.py ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """
2
+ FIGMA inference model (inference-only).
3
+
4
+ Loads the packaged FIGMA checkpoint (frozen MuQ audio encoder + frozen E5 text
5
+ encoder + two Transformer projection heads) and produces the GLOBAL audio/text
6
+ embeddings used for retrieval. Retrieval score = cosine similarity between the
7
+ global audio and text embeddings.
8
+
9
+ Note: retrieval uses only the global embeddings (mean-pooled audio frames + E5
10
+ [CLS]). The frame/token projection path is used during training only.
11
+
12
+ License: this model bundles MuQ weights (CC BY-NC 4.0), so the composite model is
13
+ released for NON-COMMERCIAL research use (CC BY-NC 4.0). MuQ and E5 are attributed.
14
+ """
15
+
16
+ import torch
17
+ import torch.nn as nn
18
+ import torch.nn.functional as F
19
+ from torch.nn import TransformerEncoder, TransformerEncoderLayer
20
+ from transformers import AutoModel, AutoTokenizer
21
+ from muq import MuQ
22
+
23
+ TEXT_ENCODER = "intfloat/multilingual-e5-large-instruct"
24
+ AUDIO_ENCODER = "OpenMuQ/MuQ-large-msd-iter"
25
+ SAMPLE_RATE = 24000
26
+ CLIP_SECONDS = 10.0
27
+
28
+
29
+ class ProjectionHead(nn.Module):
30
+ def __init__(self, input_dim, hidden_dim=512, output_dim=512):
31
+ super().__init__()
32
+ layer = TransformerEncoderLayer(
33
+ d_model=input_dim, nhead=8, dim_feedforward=hidden_dim,
34
+ dropout=0.1, batch_first=True,
35
+ )
36
+ self.transformer = TransformerEncoder(layer, num_layers=2)
37
+ self.output_proj = nn.Linear(input_dim, output_dim)
38
+
39
+ def forward(self, x):
40
+ if x.dim() == 2:
41
+ x = x.unsqueeze(1)
42
+ x = self.transformer(x)
43
+ x = x.squeeze(1)
44
+ else:
45
+ x = self.transformer(x)
46
+ return self.output_proj(x)
47
+
48
+
49
+ class Figma(nn.Module):
50
+ """Submodule names (muq / text_encoder / audio_proj / text_proj) match the
51
+ training checkpoint, so its state_dict loads directly."""
52
+
53
+ def __init__(self, audio_feat_dim=1024):
54
+ super().__init__()
55
+ self.muq = MuQ.from_pretrained(AUDIO_ENCODER).eval().requires_grad_(False)
56
+ self.text_encoder = AutoModel.from_pretrained(TEXT_ENCODER)
57
+ for p in self.text_encoder.parameters():
58
+ p.requires_grad = False
59
+ self.audio_proj = ProjectionHead(input_dim=audio_feat_dim)
60
+ self.text_proj = ProjectionHead(input_dim=self.text_encoder.config.hidden_size)
61
+
62
+ @torch.no_grad()
63
+ def encode_audio(self, wavs):
64
+ """wavs: [B, 1, samples] float tensor at 24 kHz -> [B, 512] L2-normalized."""
65
+ wavs = torch.nan_to_num(wavs, nan=0.0, posinf=0.0, neginf=0.0)
66
+ with torch.amp.autocast("cuda", enabled=False):
67
+ a_seq = self.muq(wavs).last_hidden_state
68
+ a_clip = a_seq.mean(dim=1)
69
+ return F.normalize(self.audio_proj(a_clip), dim=-1, eps=1e-8)
70
+
71
+ @torch.no_grad()
72
+ def encode_text(self, text_inputs):
73
+ """text_inputs: dict of tokenized tensors -> [B, 512] L2-normalized."""
74
+ t_seq = self.text_encoder(**text_inputs).last_hidden_state
75
+ t_cls = t_seq[:, 0, :]
76
+ return F.normalize(self.text_proj(t_cls), dim=-1, eps=1e-8)
77
+
78
+ @classmethod
79
+ def from_checkpoint(cls, ckpt_path, device="cpu"):
80
+ model = cls().to(device).eval()
81
+ ckpt = torch.load(ckpt_path, map_location="cpu", weights_only=False)
82
+ sd = ckpt.get("state_dict", ckpt)
83
+ missing, _ = model.load_state_dict(sd, strict=False)
84
+ bad = [k for k in missing if k.startswith(("audio_proj", "text_proj"))]
85
+ if bad:
86
+ raise RuntimeError(f"Projection-head weights missing from checkpoint: {bad}")
87
+ return model
88
+
89
+
90
+ def get_tokenizer():
91
+ return AutoTokenizer.from_pretrained(TEXT_ENCODER)
inference.py ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ """
3
+ FIGMA inference: encode music and text into a shared space and retrieve.
4
+
5
+ Examples
6
+ --------
7
+ # Text -> Audio: rank a folder of .wav clips by a text query
8
+ python inference.py --checkpoint figma.ckpt \
9
+ --audio_dir ./clips --query "a song in F minor at 120 BPM in 4/4 time" --topk 5
10
+
11
+ # Dump audio embeddings for a folder of clips
12
+ python inference.py --checkpoint figma.ckpt --audio_dir ./clips --save_embeddings emb.pt
13
+ """
14
+
15
+ import os
16
+ import glob
17
+ import argparse
18
+
19
+ import torch
20
+ import librosa
21
+
22
+ from figma_model import Figma, get_tokenizer, SAMPLE_RATE, CLIP_SECONDS
23
+
24
+
25
+ def load_audio_batch(paths, clip_seconds, sr=SAMPLE_RATE):
26
+ fixed = int(sr * clip_seconds)
27
+ out = torch.zeros(len(paths), 1, fixed)
28
+ for i, p in enumerate(paths):
29
+ wav, _ = librosa.load(p, sr=sr)
30
+ wav = torch.tensor(wav, dtype=torch.float32)
31
+ take = min(wav.numel(), fixed)
32
+ out[i, 0, :take] = wav[:take]
33
+ return out
34
+
35
+
36
+ @torch.no_grad()
37
+ def embed_audios(model, paths, clip_seconds, device, batch_size=32):
38
+ embs = []
39
+ for i in range(0, len(paths), batch_size):
40
+ batch = load_audio_batch(paths[i:i + batch_size], clip_seconds).to(device)
41
+ embs.append(model.encode_audio(batch).cpu())
42
+ return torch.cat(embs) if embs else torch.empty(0)
43
+
44
+
45
+ @torch.no_grad()
46
+ def embed_texts(model, tokenizer, texts, device, max_len=128, batch_size=64):
47
+ embs = []
48
+ for i in range(0, len(texts), batch_size):
49
+ tok = tokenizer(texts[i:i + batch_size], return_tensors="pt",
50
+ padding="max_length", truncation=True, max_length=max_len)
51
+ tok = {k: v.to(device) for k, v in tok.items()}
52
+ embs.append(model.encode_text(tok).cpu())
53
+ return torch.cat(embs) if embs else torch.empty(0)
54
+
55
+
56
+ def main():
57
+ ap = argparse.ArgumentParser()
58
+ ap.add_argument("--checkpoint", required=True, help="Path to the FIGMA checkpoint (.ckpt)")
59
+ ap.add_argument("--audio_dir", required=True, help="Folder of audio files")
60
+ ap.add_argument("--audio_glob", default="*.wav")
61
+ ap.add_argument("--query", help="Text query for text->audio retrieval")
62
+ ap.add_argument("--topk", type=int, default=5)
63
+ ap.add_argument("--save_embeddings", help="Path to save {paths, audio_emb} .pt")
64
+ ap.add_argument("--clip_seconds", type=float, default=CLIP_SECONDS)
65
+ ap.add_argument("--gpu", type=int, default=0)
66
+ args = ap.parse_args()
67
+
68
+ device = torch.device(f"cuda:{args.gpu}" if torch.cuda.is_available() else "cpu")
69
+ model = Figma.from_checkpoint(args.checkpoint, device=device)
70
+ tokenizer = get_tokenizer()
71
+
72
+ paths = sorted(glob.glob(os.path.join(args.audio_dir, args.audio_glob)))
73
+ if not paths:
74
+ raise SystemExit(f"No audio matching {args.audio_glob} in {args.audio_dir}")
75
+ print(f"Encoding {len(paths)} audio clips at {args.clip_seconds}s ...")
76
+ audio_emb = embed_audios(model, paths, args.clip_seconds, device)
77
+
78
+ if args.save_embeddings:
79
+ torch.save({"paths": paths, "audio_emb": audio_emb}, args.save_embeddings)
80
+ print(f"Saved embeddings -> {args.save_embeddings}")
81
+
82
+ if args.query:
83
+ q = embed_texts(model, tokenizer, [args.query], device)[0]
84
+ scores = audio_emb @ q
85
+ topk = scores.topk(min(args.topk, len(paths)))
86
+ print(f"\nTop {topk.indices.numel()} matches for: {args.query!r}")
87
+ for rank, (idx, sc) in enumerate(zip(topk.indices.tolist(), topk.values.tolist()), 1):
88
+ print(f" {rank:>2}. {sc:.4f} {os.path.basename(paths[idx])}")
89
+
90
+
91
+ if __name__ == "__main__":
92
+ main()
requirements.txt ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ torch>=2.2
2
+ transformers>=4.40
3
+ librosa>=0.10
4
+ soundfile
5
+ numpy
6
+ # MuQ audio encoder — install with: pip3 install muq
7
+ muq