// S3 to HF Bucket Importer - Application Logic
// Everything runs in the browser. No data is stored locally.
import { oauthLoginUrl, oauthHandleRedirectIfPresent } from "@huggingface/hub";
import { S3Client, ListObjectsV2Command } from "@aws-sdk/client-s3";
// ============================================================
// State
// ============================================================
const state = {
// Auth
oauthResult: null,
username: "",
orgs: [],
// S3
s3Client: null,
s3Bucket: "",
s3Prefix: "",
s3Region: "",
s3Endpoint: "",
awsAccessKeyId: "",
awsSecretAccessKey: "",
// Tree
tree: [], // top-level nodes
allFiles: new Map(), // key -> { size, selected }
allFolders: new Map(), // prefix -> { selected, children loaded }
totalFiles: 0,
totalSize: 0,
selectedFiles: 0,
selectedSize: 0,
// Mode
corsFallback: false,
// Hardware
hardware: [], // fetched from API, CPU-only
// Job
jobId: null,
jobNamespace: null,
};
// ============================================================
// DOM references
// ============================================================
const $ = (id) => document.getElementById(id);
const dom = {
signInScreen: $("sign-in-screen"),
signInBtn: $("btn-sign-in"),
signInError: $("sign-in-error"),
appHeader: $("app-header"),
userInfo: $("user-info"),
mainApp: $("main-app"),
// S3
s3Section: $("s3-section"),
s3Status: $("s3-status"),
awsAccessKey: $("aws-access-key"),
awsSecretKey: $("aws-secret-key"),
awsRegion: $("aws-region"),
s3Bucket: $("s3-bucket"),
s3Endpoint: $("s3-endpoint"),
s3Prefix: $("s3-prefix"),
btnConnect: $("btn-connect"),
s3ConnectError: $("s3-connect-error"),
// Browser
browserSection: $("browser-section"),
browserNormal: $("browser-normal"),
browserCorsFallback: $("browser-cors-fallback"),
treeContainer: $("tree-container"),
statSelectedCount: $("stat-selected-count"),
statSelectedSize: $("stat-selected-size"),
statTotalCount: $("stat-total-count"),
statTotalSize: $("stat-total-size"),
btnSelectAll: $("btn-select-all"),
btnDeselectAll: $("btn-deselect-all"),
manualInclude: $("manual-include"),
manualExclude: $("manual-exclude"),
btnContinueImport: $("btn-continue-import"),
// Import
importSection: $("import-section"),
destBucket: $("dest-bucket"),
destPrefix: $("dest-prefix"),
destPrivate: $("dest-private"),
destPreview: $("dest-preview"),
jobFlavor: $("job-flavor"),
jobWorkers: $("job-workers"),
jobTimeout: $("job-timeout"),
patternsInfo: $("patterns-info"),
patternsPreview: $("patterns-preview"),
importError: $("import-error"),
btnStartImport: $("btn-start-import"),
// Status
statusSection: $("status-section"),
jobStatusIcon: $("job-status-icon"),
jobStatusBadge: $("job-status-badge"),
jobStatusTitle: $("job-status-title"),
jobIdDisplay: $("job-id-display"),
jobLink: $("job-link"),
bucketLink: $("bucket-link"),
jobStatusMessage: $("job-status-message"),
btnNewImport: $("btn-new-import"),
};
// ============================================================
// Utilities
// ============================================================
function formatSize(bytes) {
if (bytes === 0) return "0 B";
const units = ["B", "KB", "MB", "GB", "TB"];
const i = Math.floor(Math.log(bytes) / Math.log(1024));
const val = bytes / Math.pow(1024, i);
return `${val < 10 ? val.toFixed(2) : val < 100 ? val.toFixed(1) : Math.round(val)} ${units[i]}`;
}
function shellEscape(str) {
return "'" + str.replace(/'/g, "'\\''") + "'";
}
function show(el) { el.classList.remove("hidden"); }
function hide(el) { el.classList.add("hidden"); }
function setLoading(btn, loading, text) {
if (loading) {
btn._origText = btn.textContent;
btn.disabled = true;
btn.innerHTML = ` ${text || "Loading..."}`;
} else {
btn.disabled = false;
btn.textContent = btn._origText || text || "Submit";
}
}
// ============================================================
// OAuth
// ============================================================
async function initOAuth() {
try {
const result = await oauthHandleRedirectIfPresent();
if (result) {
state.oauthResult = result;
state.username = result.userInfo.preferred_username || result.userInfo.name || result.userInfo.sub;
await Promise.all([fetchUserOrgs(), fetchHardware()]);
showLoggedIn();
return;
}
} catch (err) {
console.error("OAuth redirect handling failed:", err);
}
// Not logged in - show sign-in button
dom.signInBtn.disabled = false;
dom.signInBtn.addEventListener("click", async () => {
try {
dom.signInBtn.disabled = true;
const url = await oauthLoginUrl({ scopes: "openid profile manage-repos jobs" });
window.location.href = url;
} catch (err) {
show(dom.signInError);
dom.signInError.textContent = `Sign-in failed: ${err.message}`;
dom.signInBtn.disabled = false;
}
});
}
async function fetchUserOrgs() {
try {
const resp = await fetch("https://huggingface.co/api/whoami-v2", {
headers: { Authorization: `Bearer ${state.oauthResult.accessToken}` },
});
if (resp.ok) {
const data = await resp.json();
state.orgs = (data.orgs || []).map((o) => o.name);
}
} catch {
// Non-critical, ignore
}
}
async function fetchHardware() {
try {
const resp = await fetch("https://huggingface.co/api/jobs/hardware");
if (resp.ok) {
const data = await resp.json();
// Keep only CPU hardware (no accelerator)
state.hardware = data.filter((h) => h.accelerator === null);
populateHardwareSelect();
}
} catch {
// Fallback: keep the placeholder, user can still type a flavor
}
}
function populateHardwareSelect() {
dom.jobFlavor.innerHTML = "";
for (const hw of state.hardware) {
const opt = document.createElement("option");
opt.value = hw.name;
// Parse vCPU count from "2 vCPU" string
const vcpus = parseInt(hw.cpu) || 2;
opt.dataset.vcpus = vcpus;
// Format cost per hour from per-minute cost
const costPerHour = (hw.unitCostUSD * 60).toFixed(2);
opt.textContent = `${hw.prettyName} - ${hw.cpu}, ${hw.ram} RAM ($${costPerHour}/hr)`;
dom.jobFlavor.appendChild(opt);
}
// Select first option and update workers accordingly
if (state.hardware.length > 0) {
dom.jobFlavor.selectedIndex = 0;
updateWorkersFromHardware();
}
}
function updateWorkersFromHardware() {
const selected = dom.jobFlavor.selectedOptions[0];
if (!selected) return;
const vcpus = parseInt(selected.dataset.vcpus) || 2;
// Default workers = 2 * vCPUs - 2 (clamped to 1-16)
dom.jobWorkers.value = Math.min(Math.max(2 * vcpus - 2, 1), 16);
}
function showLoggedIn() {
hide(dom.signInScreen);
show(dom.appHeader);
show(dom.mainApp);
const avatar = state.oauthResult.userInfo.picture || "";
const name = state.username;
dom.userInfo.innerHTML = `
${avatar ? `
` : ""}
${name}
`;
document.getElementById("btn-sign-out").addEventListener("click", () => {
// Clear state and reload
window.location.href = window.location.pathname;
});
}
// ============================================================
// S3 Connection
// ============================================================
async function connectS3() {
const accessKey = dom.awsAccessKey.value.trim();
const secretKey = dom.awsSecretKey.value.trim();
const region = dom.awsRegion.value.trim() || "us-east-1";
const bucket = dom.s3Bucket.value.trim();
const endpoint = dom.s3Endpoint.value.trim();
const prefix = dom.s3Prefix.value.trim();
// Validation
if (!bucket) {
dom.s3ConnectError.textContent = "Please provide an S3 bucket name.";
dom.s3ConnectError.style.color = "var(--error)";
return;
}
dom.s3ConnectError.textContent = "";
setLoading(dom.btnConnect, true, "Connecting...");
// Store credentials for later use in Job
state.awsAccessKeyId = accessKey;
state.awsSecretAccessKey = secretKey;
state.s3Region = region;
state.s3Endpoint = endpoint;
state.s3Bucket = bucket;
state.s3Prefix = prefix.endsWith("/") || prefix === "" ? prefix : prefix + "/";
// Create S3 client
const clientConfig = {
region,
};
if (accessKey && secretKey) {
clientConfig.credentials = {
accessKeyId: accessKey,
secretAccessKey: secretKey,
};
} else {
// Anonymous access for public buckets
clientConfig.signer = { sign: async (request) => request };
}
if (endpoint) {
clientConfig.endpoint = endpoint;
clientConfig.forcePathStyle = true;
}
state.s3Client = new S3Client(clientConfig);
// Try listing
try {
const result = await listS3Prefix(state.s3Prefix);
state.corsFallback = false;
// Success - show browser
show(dom.browserSection);
show(dom.browserNormal);
hide(dom.browserCorsFallback);
dom.s3Status.textContent = "Connected";
dom.s3Status.className = "card-status connected";
show(dom.s3Status);
// Build initial tree
resetTree();
renderTreeLevel(result, state.s3Prefix, dom.treeContainer, 0);
updateStats();
} catch (err) {
console.error("S3 listing error:", err);
const msg = err.message || String(err);
// Check if it's a credentials error vs CORS
if (msg.includes("403") || msg.includes("AccessDenied") || msg.includes("InvalidAccessKeyId") || msg.includes("SignatureDoesNotMatch")) {
dom.s3ConnectError.textContent = "Invalid credentials or access denied. Check your Access Key and Secret.";
dom.s3ConnectError.style.color = "var(--error)";
setLoading(dom.btnConnect, false, "Connect & Browse");
return;
}
if (msg.includes("404") || msg.includes("NoSuchBucket")) {
dom.s3ConnectError.textContent = "Bucket not found. Check the bucket name and region.";
dom.s3ConnectError.style.color = "var(--error)";
setLoading(dom.btnConnect, false, "Connect & Browse");
return;
}
// CORS or network error - fallback mode
state.corsFallback = true;
show(dom.browserSection);
hide(dom.browserNormal);
show(dom.browserCorsFallback);
dom.s3Status.textContent = "Connected (no browse)";
dom.s3Status.className = "card-status connected";
show(dom.s3Status);
}
setLoading(dom.btnConnect, false, "Connect & Browse");
}
async function listS3Prefix(prefix, continuationToken) {
const params = {
Bucket: state.s3Bucket,
Prefix: prefix || undefined,
Delimiter: "/",
MaxKeys: 1000,
};
if (continuationToken) {
params.ContinuationToken = continuationToken;
}
const command = new ListObjectsV2Command(params);
const response = await state.s3Client.send(command);
return {
folders: (response.CommonPrefixes || []).map((cp) => cp.Prefix),
files: (response.Contents || [])
.filter((obj) => obj.Key !== prefix) // exclude the prefix itself
.map((obj) => ({
key: obj.Key,
size: obj.Size || 0,
lastModified: obj.LastModified,
})),
isTruncated: response.IsTruncated,
continuationToken: response.NextContinuationToken,
};
}
// ============================================================
// Tree View
// ============================================================
function resetTree() {
state.tree = [];
state.allFiles.clear();
state.allFolders.clear();
state.totalFiles = 0;
state.totalSize = 0;
state.selectedFiles = 0;
state.selectedSize = 0;
dom.treeContainer.innerHTML = "";
}
function renderTreeLevel(result, parentPrefix, container, depth) {
// Render folders
for (const folderKey of result.folders) {
const relativeName = folderKey.slice(parentPrefix.length);
state.allFolders.set(folderKey, { selected: true, loaded: false });
const node = createTreeNode({
key: folderKey,
name: relativeName.replace(/\/$/, ""),
isFolder: true,
size: 0,
depth,
});
container.appendChild(node);
}
// Render files
for (const file of result.files) {
const relativeName = file.key.slice(parentPrefix.length);
state.allFiles.set(file.key, { size: file.size, selected: true });
state.totalFiles++;
state.totalSize += file.size;
state.selectedFiles++;
state.selectedSize += file.size;
const node = createTreeNode({
key: file.key,
name: relativeName,
isFolder: false,
size: file.size,
depth,
});
container.appendChild(node);
}
// Load more button
if (result.isTruncated) {
const loadMore = document.createElement("div");
loadMore.className = "tree-load-more";
loadMore.style.paddingLeft = `${(depth + 1) * 24 + 12}px`;
loadMore.textContent = "Load more...";
loadMore.addEventListener("click", async () => {
loadMore.textContent = "Loading...";
try {
const moreResult = await listS3Prefix(parentPrefix, result.continuationToken);
loadMore.remove();
renderTreeLevel(moreResult, parentPrefix, container, depth);
updateStats();
} catch (err) {
loadMore.textContent = `Error: ${err.message}. Click to retry.`;
}
});
container.appendChild(loadMore);
}
}
function createTreeNode({ key, name, isFolder, size, depth }) {
const row = document.createElement("div");
row.className = "tree-node";
row.dataset.key = key;
row.dataset.isFolder = isFolder;
// Indent
const indent = document.createElement("span");
indent.className = "indent";
indent.style.width = `${depth * 24}px`;
row.appendChild(indent);
// Expand icon (folders only)
const expandIcon = document.createElement("span");
expandIcon.className = `expand-icon ${isFolder ? "" : "hidden"}`;
expandIcon.textContent = "\u25B6"; // right triangle
row.appendChild(expandIcon);
// Checkbox
const checkbox = document.createElement("input");
checkbox.type = "checkbox";
checkbox.className = "node-checkbox";
checkbox.checked = true; // selected by default
checkbox.addEventListener("change", (e) => {
e.stopPropagation();
toggleSelection(key, isFolder, checkbox.checked);
});
checkbox.addEventListener("click", (e) => e.stopPropagation());
row.appendChild(checkbox);
// Icon
const icon = document.createElement("span");
icon.className = "node-icon";
icon.textContent = isFolder ? "\uD83D\uDCC1" : "\uD83D\uDCC4";
row.appendChild(icon);
// Name
const nameSpan = document.createElement("span");
nameSpan.className = "node-name";
nameSpan.textContent = name;
row.appendChild(nameSpan);
// Meta (size for files)
const meta = document.createElement("span");
meta.className = "node-meta";
meta.textContent = isFolder ? "" : formatSize(size);
row.appendChild(meta);
// Click handler for folders (expand/collapse)
if (isFolder) {
row.addEventListener("click", () => expandFolder(key, row, depth));
}
return row;
}
async function expandFolder(folderKey, row, depth) {
const expandIcon = row.querySelector(".expand-icon");
const folderState = state.allFolders.get(folderKey);
// Toggle collapse
if (expandIcon.classList.contains("expanded")) {
expandIcon.classList.remove("expanded");
// Hide children
let next = row.nextElementSibling;
while (next && next.dataset.parentFolder === folderKey) {
next.style.display = "none";
next = next.nextElementSibling;
}
return;
}
expandIcon.classList.add("expanded");
// If already loaded, just show children
if (folderState && folderState.loaded) {
let next = row.nextElementSibling;
while (next && next.dataset.parentFolder === folderKey) {
next.style.display = "";
next = next.nextElementSibling;
}
return;
}
// Load children
const loadingRow = document.createElement("div");
loadingRow.className = "tree-node loading";
loadingRow.dataset.parentFolder = folderKey;
loadingRow.innerHTML = `
Loading...
`;
row.after(loadingRow);
try {
const result = await listS3Prefix(folderKey);
loadingRow.remove();
folderState.loaded = true;
// Update meta with count
const meta = row.querySelector(".node-meta");
const totalItems = result.folders.length + result.files.length + (result.isTruncated ? "+" : "");
meta.textContent = `${totalItems} items`;
// Create a container fragment
const fragment = document.createDocumentFragment();
const tempContainer = document.createElement("div");
renderTreeLevel(result, folderKey, tempContainer, depth + 1);
// Mark all children with parent folder for collapse/expand
for (const child of tempContainer.children) {
child.dataset.parentFolder = folderKey;
}
// Insert after the folder row
let insertAfter = row;
while (tempContainer.firstChild) {
const child = tempContainer.firstChild;
child.dataset.parentFolder = folderKey;
insertAfter.after(child);
insertAfter = child;
}
// If folder is selected, ensure children are selected too
if (folderState.selected) {
// Children are already selected by default from renderTreeLevel
}
updateStats();
} catch (err) {
loadingRow.querySelector(".node-name").textContent = `Error: ${err.message}`;
loadingRow.querySelector(".node-icon").innerHTML = "";
}
}
function toggleSelection(key, isFolder, selected) {
if (isFolder) {
const folderState = state.allFolders.get(key);
if (folderState) folderState.selected = selected;
// Toggle all loaded children
for (const [fileKey, fileState] of state.allFiles) {
if (fileKey.startsWith(key)) {
const wasSelected = fileState.selected;
fileState.selected = selected;
if (wasSelected !== selected) {
state.selectedFiles += selected ? 1 : -1;
state.selectedSize += selected ? fileState.size : -fileState.size;
}
// Update checkbox in DOM
const row = dom.treeContainer.querySelector(`[data-key="${CSS.escape(fileKey)}"]`);
if (row) row.querySelector(".node-checkbox").checked = selected;
}
}
// Toggle child folders
for (const [folderKeyChild, folderState2] of state.allFolders) {
if (folderKeyChild.startsWith(key) && folderKeyChild !== key) {
folderState2.selected = selected;
const row = dom.treeContainer.querySelector(`[data-key="${CSS.escape(folderKeyChild)}"]`);
if (row) row.querySelector(".node-checkbox").checked = selected;
}
}
} else {
const fileState = state.allFiles.get(key);
if (fileState) {
const wasSelected = fileState.selected;
fileState.selected = selected;
if (wasSelected !== selected) {
state.selectedFiles += selected ? 1 : -1;
state.selectedSize += selected ? fileState.size : -fileState.size;
}
}
}
updateStats();
}
function selectAll(selected) {
for (const [key, fileState] of state.allFiles) {
fileState.selected = selected;
}
for (const [key, folderState] of state.allFolders) {
folderState.selected = selected;
}
// Update all checkboxes
dom.treeContainer.querySelectorAll(".node-checkbox").forEach((cb) => {
cb.checked = selected;
});
// Recalculate
if (selected) {
state.selectedFiles = state.totalFiles;
state.selectedSize = state.totalSize;
} else {
state.selectedFiles = 0;
state.selectedSize = 0;
}
updateStats();
}
function updateStats() {
dom.statSelectedCount.textContent = state.selectedFiles.toLocaleString();
dom.statSelectedSize.textContent = formatSize(state.selectedSize);
dom.statTotalCount.textContent = state.totalFiles.toLocaleString();
dom.statTotalSize.textContent = formatSize(state.totalSize);
}
// ============================================================
// Import Configuration
// ============================================================
function showImportSection() {
show(dom.importSection);
// Default destination bucket to the S3 bucket name if not already set
if (!dom.destBucket.value.trim() && state.s3Bucket) {
dom.destBucket.value = state.s3Bucket;
updateDestPreview();
}
dom.importSection.scrollIntoView({ behavior: "smooth", block: "start" });
// Update patterns preview
updatePatternsPreview();
}
function updateDestPreview() {
const bucket = dom.destBucket.value.trim();
if (!bucket) {
hide(dom.destPreview);
return;
}
const fullBucket = bucket.includes("/") ? bucket : `${state.username}/${bucket}`;
const prefix = dom.destPrefix.value.trim();
const path = prefix ? `${fullBucket}/${prefix.replace(/\/$/, "")}` : fullBucket;
dom.destPreview.textContent = `hf://buckets/${path}`;
show(dom.destPreview);
}
function computePatterns() {
if (state.corsFallback) {
const includes = dom.manualInclude.value.trim().split("\n").filter(Boolean);
const excludes = dom.manualExclude.value.trim().split("\n").filter(Boolean);
return { include: includes, exclude: excludes };
}
// All files selected?
if (state.selectedFiles === state.totalFiles) {
return { include: [], exclude: [] };
}
// No files selected?
if (state.selectedFiles === 0) {
return { include: ["__nothing__"], exclude: [] };
}
// Compute selected/deselected
const selected = [];
const deselected = [];
for (const [key, fileState] of state.allFiles) {
const relKey = key.startsWith(state.s3Prefix) ? key.slice(state.s3Prefix.length) : key;
if (fileState.selected) {
selected.push(relKey);
} else {
deselected.push(relKey);
}
}
// Use whichever set is smaller
if (deselected.length <= selected.length) {
return { include: [], exclude: deselected };
} else {
// Try to group by folder
const patterns = compactPatterns(selected);
return { include: patterns, exclude: [] };
}
}
function compactPatterns(keys) {
// Group files by their parent folder and check if entire folder is selected
const folderCounts = new Map();
const folderTotals = new Map();
for (const key of keys) {
const parts = key.split("/");
if (parts.length > 1) {
const folder = parts.slice(0, -1).join("/") + "/";
folderCounts.set(folder, (folderCounts.get(folder) || 0) + 1);
}
}
// Count total files per folder
for (const [fileKey] of state.allFiles) {
const relKey = fileKey.startsWith(state.s3Prefix) ? fileKey.slice(state.s3Prefix.length) : fileKey;
const parts = relKey.split("/");
if (parts.length > 1) {
const folder = parts.slice(0, -1).join("/") + "/";
folderTotals.set(folder, (folderTotals.get(folder) || 0) + 1);
}
}
const patterns = [];
const coveredKeys = new Set();
// If entire folder is selected, use folder/*
for (const [folder, count] of folderCounts) {
if (count === (folderTotals.get(folder) || 0)) {
patterns.push(`${folder}*`);
for (const key of keys) {
if (key.startsWith(folder)) coveredKeys.add(key);
}
}
}
// Add remaining individual files
for (const key of keys) {
if (!coveredKeys.has(key)) {
patterns.push(key);
}
}
return patterns;
}
function updatePatternsPreview() {
const patterns = computePatterns();
const s3Uri = `s3://${state.s3Bucket}/${state.s3Prefix}`;
const destBucket = dom.destBucket.value.trim();
const nsDest = destBucket.includes("/") ? destBucket : `${state.username}/${destBucket}`;
const destPrefix = dom.destPrefix.value.trim();
const dest = destPrefix ? `hf://buckets/${nsDest}/${destPrefix.replace(/\/$/, "")}` : `hf://buckets/${nsDest}`;
const workers = dom.jobWorkers.value || "8";
let cmd = `hf buckets import ${shellEscape(s3Uri)} ${shellEscape(dest)}`;
if (patterns.include.length > 0) {
for (const p of patterns.include) {
cmd += ` --include ${shellEscape(p)}`;
}
}
if (patterns.exclude.length > 0) {
for (const p of patterns.exclude) {
cmd += ` --exclude ${shellEscape(p)}`;
}
}
cmd += ` --workers ${workers}`;
cmd += ` --buffer-size 49g`;
dom.patternsPreview.textContent = cmd;
show(dom.patternsInfo);
}
// ============================================================
// Bucket Creation
// ============================================================
async function ensureBucketExists(namespace, bucketName, isPrivate) {
const token = state.oauthResult.accessToken;
// Use "me" for the current user's namespace, matching the Python SDK behavior
const ns = namespace === state.username ? "me" : namespace;
const resp = await fetch(`https://huggingface.co/api/buckets/${ns}/${bucketName}`, {
method: "POST",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
},
body: JSON.stringify({ private: isPrivate }),
});
if (resp.ok || resp.status === 409) {
// Created or already exists
return true;
}
const errorText = await resp.text();
throw new Error(`Failed to create bucket: ${resp.status} - ${errorText}`);
}
// ============================================================
// Job Submission
// ============================================================
async function submitImportJob() {
const destBucketInput = dom.destBucket.value.trim();
if (!destBucketInput) {
showImportError("Please provide a destination bucket name.");
return;
}
// Parse namespace/bucket
let namespace, bucketName;
if (destBucketInput.includes("/")) {
[namespace, bucketName] = destBucketInput.split("/", 2);
} else {
namespace = state.username;
bucketName = destBucketInput;
}
const destPrefix = dom.destPrefix.value.trim().replace(/\/$/, "");
const fullDest = destPrefix ? `hf://buckets/${namespace}/${bucketName}/${destPrefix}` : `hf://buckets/${namespace}/${bucketName}`;
const s3Uri = `s3://${state.s3Bucket}/${state.s3Prefix}`;
const flavor = dom.jobFlavor.value;
const workers = parseInt(dom.jobWorkers.value) || 8;
const timeout = parseInt(dom.jobTimeout.value) || 7200;
const token = state.oauthResult.accessToken;
hideImportError();
setLoading(dom.btnStartImport, true, "Creating bucket & submitting job...");
try {
// Step 1: Ensure bucket exists
const isPrivate = dom.destPrivate.checked;
await ensureBucketExists(namespace, bucketName, isPrivate);
// Step 2: Build command
const patterns = computePatterns();
let script = "pip install -q 'huggingface_hub[s3] @ git+https://github.com/huggingface/huggingface_hub.git@cursor/s3-to-hf-bucket-ingestion-f144'";
script += ` && hf buckets import ${shellEscape(s3Uri)} ${shellEscape(fullDest)}`;
if (patterns.include.length > 0) {
for (const p of patterns.include) {
script += ` --include ${shellEscape(p)}`;
}
}
if (patterns.exclude.length > 0) {
for (const p of patterns.exclude) {
script += ` --exclude ${shellEscape(p)}`;
}
}
script += ` --workers ${workers}`;
script += ` --buffer-size 49g`;
// Step 3: Submit job
const secrets = { HF_TOKEN: token };
if (state.awsAccessKeyId) secrets.AWS_ACCESS_KEY_ID = state.awsAccessKeyId;
if (state.awsSecretAccessKey) secrets.AWS_SECRET_ACCESS_KEY = state.awsSecretAccessKey;
const jobPayload = {
dockerImage: "python:3.12",
command: ["bash", "-c", script],
arguments: [],
flavor,
timeout,
environment: {
AWS_DEFAULT_REGION: state.s3Region || "us-east-1",
},
secrets,
};
if (state.s3Endpoint) {
jobPayload.environment.AWS_ENDPOINT_URL = state.s3Endpoint;
}
const resp = await fetch(`https://huggingface.co/api/jobs/${namespace}`, {
method: "POST",
headers: {
Authorization: `Bearer ${token}`,
"Content-Type": "application/json",
},
body: JSON.stringify(jobPayload),
});
if (!resp.ok) {
const errorText = await resp.text();
throw new Error(`Job creation failed (${resp.status}): ${errorText}`);
}
const jobData = await resp.json();
state.jobId = jobData.id;
state.jobNamespace = namespace;
// Show status
showJobStatus(jobData);
pollJobStatus();
} catch (err) {
showImportError(err.message);
setLoading(dom.btnStartImport, false, "Start Import");
}
}
function showImportError(msg) {
dom.importError.textContent = msg;
show(dom.importError);
}
function hideImportError() {
hide(dom.importError);
}
// ============================================================
// Job Status
// ============================================================
function showJobStatus(jobData) {
hide(dom.s3Section);
hide(dom.browserSection);
hide(dom.importSection);
show(dom.statusSection);
const jobUrl = `https://huggingface.co/jobs/${state.jobNamespace}/${jobData.id}`;
dom.jobIdDisplay.textContent = `Job ID: ${jobData.id}`;
dom.jobLink.href = jobUrl;
// Build bucket URL from destination
const destBucketInput = dom.destBucket.value.trim();
let bucketNs, bucketName;
if (destBucketInput.includes("/")) {
[bucketNs, bucketName] = destBucketInput.split("/", 2);
} else {
bucketNs = state.username;
bucketName = destBucketInput;
}
dom.bucketLink.href = `https://huggingface.co/buckets/${bucketNs}/${bucketName}`;
updateJobStatusUI(jobData.status?.stage || "SCHEDULING");
}
function updateJobStatusUI(stage) {
const lowerStage = stage.toLowerCase();
dom.jobStatusBadge.textContent = stage;
dom.jobStatusBadge.className = `job-status-badge ${lowerStage}`;
switch (stage) {
case "SCHEDULING":
dom.jobStatusIcon.textContent = "\uD83D\uDE80";
dom.jobStatusTitle.textContent = "Import Job Submitted";
dom.jobStatusMessage.textContent = "Your job is being scheduled. This usually takes a few seconds.";
break;
case "RUNNING":
dom.jobStatusIcon.textContent = "\u2699\uFE0F";
dom.jobStatusTitle.textContent = "Import in Progress";
dom.jobStatusMessage.textContent = "Your data is being transferred from S3 to your HF bucket. You can close this page - the job will continue running.";
break;
case "COMPLETED":
dom.jobStatusIcon.textContent = "\u2705";
dom.jobStatusTitle.textContent = "Import Complete!";
dom.jobStatusMessage.textContent = "Your data has been successfully imported to the HF bucket.";
break;
case "ERROR":
dom.jobStatusIcon.textContent = "\u274C";
dom.jobStatusTitle.textContent = "Import Failed";
dom.jobStatusMessage.textContent = "The import job encountered an error. Check the job logs for details.";
break;
case "CANCELED":
dom.jobStatusIcon.textContent = "\u23F9\uFE0F";
dom.jobStatusTitle.textContent = "Import Canceled";
dom.jobStatusMessage.textContent = "The import job was canceled.";
break;
default:
dom.jobStatusIcon.textContent = "\u2753";
dom.jobStatusTitle.textContent = `Status: ${stage}`;
dom.jobStatusMessage.textContent = "";
}
}
async function pollJobStatus() {
if (!state.jobId) return;
const token = state.oauthResult.accessToken;
const terminalStages = new Set(["COMPLETED", "ERROR", "CANCELED", "DELETED"]);
const poll = async () => {
try {
const resp = await fetch(
`https://huggingface.co/api/jobs/${state.jobNamespace}/${state.jobId}`,
{ headers: { Authorization: `Bearer ${token}` } }
);
if (resp.ok) {
const data = await resp.json();
const stage = data.status?.stage || "UNKNOWN";
updateJobStatusUI(stage);
if (terminalStages.has(stage)) return; // stop polling
}
} catch {
// Network error, keep polling
}
setTimeout(poll, 5000);
};
// Start polling after a short delay
setTimeout(poll, 3000);
}
function resetForNewImport() {
hide(dom.statusSection);
show(dom.s3Section);
state.jobId = null;
state.jobNamespace = null;
// Reset browser and import sections
hide(dom.browserSection);
hide(dom.importSection);
hide(dom.s3Status);
setLoading(dom.btnStartImport, false, "Start Import");
}
// ============================================================
// Event Listeners
// ============================================================
function initEventListeners() {
dom.btnConnect.addEventListener("click", connectS3);
dom.btnSelectAll.addEventListener("click", () => selectAll(true));
dom.btnDeselectAll.addEventListener("click", () => selectAll(false));
dom.btnContinueImport.addEventListener("click", showImportSection);
dom.destBucket.addEventListener("input", () => {
updateDestPreview();
updatePatternsPreview();
});
dom.destPrefix.addEventListener("input", () => {
updateDestPreview();
updatePatternsPreview();
});
dom.jobFlavor.addEventListener("change", () => {
updateWorkersFromHardware();
updatePatternsPreview();
});
dom.jobWorkers.addEventListener("input", updatePatternsPreview);
dom.btnStartImport.addEventListener("click", submitImportJob);
dom.btnNewImport.addEventListener("click", resetForNewImport);
// Allow Enter key on S3 form
dom.s3Section.addEventListener("keydown", (e) => {
if (e.key === "Enter") {
e.preventDefault();
connectS3();
}
});
// Help toggle buttons
document.addEventListener("click", (e) => {
const toggle = e.target.closest(".help-toggle");
if (!toggle) return;
e.preventDefault();
const helpId = toggle.dataset.help;
if (!helpId) return;
const helpEl = document.getElementById(helpId);
if (!helpEl) return;
helpEl.classList.toggle("visible");
});
}
// ============================================================
// Init
// ============================================================
async function main() {
initEventListeners();
await initOAuth();
}
main().catch((err) => {
console.error("App initialization error:", err);
});