// S3 to HF Bucket Importer - Application Logic // Everything runs in the browser. No data is stored locally. import { oauthLoginUrl, oauthHandleRedirectIfPresent } from "@huggingface/hub"; import { S3Client, ListObjectsV2Command } from "@aws-sdk/client-s3"; // ============================================================ // State // ============================================================ const state = { // Auth oauthResult: null, username: "", orgs: [], // S3 s3Client: null, s3Bucket: "", s3Prefix: "", s3Region: "", s3Endpoint: "", awsAccessKeyId: "", awsSecretAccessKey: "", // Tree tree: [], // top-level nodes allFiles: new Map(), // key -> { size, selected } allFolders: new Map(), // prefix -> { selected, children loaded } totalFiles: 0, totalSize: 0, selectedFiles: 0, selectedSize: 0, // Mode corsFallback: false, // Hardware hardware: [], // fetched from API, CPU-only // Job jobId: null, jobNamespace: null, }; // ============================================================ // DOM references // ============================================================ const $ = (id) => document.getElementById(id); const dom = { signInScreen: $("sign-in-screen"), signInBtn: $("btn-sign-in"), signInError: $("sign-in-error"), appHeader: $("app-header"), userInfo: $("user-info"), mainApp: $("main-app"), // S3 s3Section: $("s3-section"), s3Status: $("s3-status"), awsAccessKey: $("aws-access-key"), awsSecretKey: $("aws-secret-key"), awsRegion: $("aws-region"), s3Bucket: $("s3-bucket"), s3Endpoint: $("s3-endpoint"), s3Prefix: $("s3-prefix"), btnConnect: $("btn-connect"), s3ConnectError: $("s3-connect-error"), // Browser browserSection: $("browser-section"), browserNormal: $("browser-normal"), browserCorsFallback: $("browser-cors-fallback"), treeContainer: $("tree-container"), statSelectedCount: $("stat-selected-count"), statSelectedSize: $("stat-selected-size"), statTotalCount: $("stat-total-count"), statTotalSize: $("stat-total-size"), btnSelectAll: $("btn-select-all"), btnDeselectAll: $("btn-deselect-all"), manualInclude: $("manual-include"), manualExclude: $("manual-exclude"), btnContinueImport: $("btn-continue-import"), // Import importSection: $("import-section"), destBucket: $("dest-bucket"), destPrefix: $("dest-prefix"), destPrivate: $("dest-private"), destPreview: $("dest-preview"), jobFlavor: $("job-flavor"), jobWorkers: $("job-workers"), jobTimeout: $("job-timeout"), patternsInfo: $("patterns-info"), patternsPreview: $("patterns-preview"), importError: $("import-error"), btnStartImport: $("btn-start-import"), // Status statusSection: $("status-section"), jobStatusIcon: $("job-status-icon"), jobStatusBadge: $("job-status-badge"), jobStatusTitle: $("job-status-title"), jobIdDisplay: $("job-id-display"), jobLink: $("job-link"), bucketLink: $("bucket-link"), jobStatusMessage: $("job-status-message"), btnNewImport: $("btn-new-import"), }; // ============================================================ // Utilities // ============================================================ function formatSize(bytes) { if (bytes === 0) return "0 B"; const units = ["B", "KB", "MB", "GB", "TB"]; const i = Math.floor(Math.log(bytes) / Math.log(1024)); const val = bytes / Math.pow(1024, i); return `${val < 10 ? val.toFixed(2) : val < 100 ? val.toFixed(1) : Math.round(val)} ${units[i]}`; } function shellEscape(str) { return "'" + str.replace(/'/g, "'\\''") + "'"; } function show(el) { el.classList.remove("hidden"); } function hide(el) { el.classList.add("hidden"); } function setLoading(btn, loading, text) { if (loading) { btn._origText = btn.textContent; btn.disabled = true; btn.innerHTML = ` ${text || "Loading..."}`; } else { btn.disabled = false; btn.textContent = btn._origText || text || "Submit"; } } // ============================================================ // OAuth // ============================================================ async function initOAuth() { try { const result = await oauthHandleRedirectIfPresent(); if (result) { state.oauthResult = result; state.username = result.userInfo.preferred_username || result.userInfo.name || result.userInfo.sub; await Promise.all([fetchUserOrgs(), fetchHardware()]); showLoggedIn(); return; } } catch (err) { console.error("OAuth redirect handling failed:", err); } // Not logged in - show sign-in button dom.signInBtn.disabled = false; dom.signInBtn.addEventListener("click", async () => { try { dom.signInBtn.disabled = true; const url = await oauthLoginUrl({ scopes: "openid profile manage-repos jobs" }); window.location.href = url; } catch (err) { show(dom.signInError); dom.signInError.textContent = `Sign-in failed: ${err.message}`; dom.signInBtn.disabled = false; } }); } async function fetchUserOrgs() { try { const resp = await fetch("https://huggingface.co/api/whoami-v2", { headers: { Authorization: `Bearer ${state.oauthResult.accessToken}` }, }); if (resp.ok) { const data = await resp.json(); state.orgs = (data.orgs || []).map((o) => o.name); } } catch { // Non-critical, ignore } } async function fetchHardware() { try { const resp = await fetch("https://huggingface.co/api/jobs/hardware"); if (resp.ok) { const data = await resp.json(); // Keep only CPU hardware (no accelerator) state.hardware = data.filter((h) => h.accelerator === null); populateHardwareSelect(); } } catch { // Fallback: keep the placeholder, user can still type a flavor } } function populateHardwareSelect() { dom.jobFlavor.innerHTML = ""; for (const hw of state.hardware) { const opt = document.createElement("option"); opt.value = hw.name; // Parse vCPU count from "2 vCPU" string const vcpus = parseInt(hw.cpu) || 2; opt.dataset.vcpus = vcpus; // Format cost per hour from per-minute cost const costPerHour = (hw.unitCostUSD * 60).toFixed(2); opt.textContent = `${hw.prettyName} - ${hw.cpu}, ${hw.ram} RAM ($${costPerHour}/hr)`; dom.jobFlavor.appendChild(opt); } // Select first option and update workers accordingly if (state.hardware.length > 0) { dom.jobFlavor.selectedIndex = 0; updateWorkersFromHardware(); } } function updateWorkersFromHardware() { const selected = dom.jobFlavor.selectedOptions[0]; if (!selected) return; const vcpus = parseInt(selected.dataset.vcpus) || 2; // Default workers = 2 * vCPUs - 2 (clamped to 1-16) dom.jobWorkers.value = Math.min(Math.max(2 * vcpus - 2, 1), 16); } function showLoggedIn() { hide(dom.signInScreen); show(dom.appHeader); show(dom.mainApp); const avatar = state.oauthResult.userInfo.picture || ""; const name = state.username; dom.userInfo.innerHTML = ` ${avatar ? `${name}` : ""} ${name} `; document.getElementById("btn-sign-out").addEventListener("click", () => { // Clear state and reload window.location.href = window.location.pathname; }); } // ============================================================ // S3 Connection // ============================================================ async function connectS3() { const accessKey = dom.awsAccessKey.value.trim(); const secretKey = dom.awsSecretKey.value.trim(); const region = dom.awsRegion.value.trim() || "us-east-1"; const bucket = dom.s3Bucket.value.trim(); const endpoint = dom.s3Endpoint.value.trim(); const prefix = dom.s3Prefix.value.trim(); // Validation if (!bucket) { dom.s3ConnectError.textContent = "Please provide an S3 bucket name."; dom.s3ConnectError.style.color = "var(--error)"; return; } dom.s3ConnectError.textContent = ""; setLoading(dom.btnConnect, true, "Connecting..."); // Store credentials for later use in Job state.awsAccessKeyId = accessKey; state.awsSecretAccessKey = secretKey; state.s3Region = region; state.s3Endpoint = endpoint; state.s3Bucket = bucket; state.s3Prefix = prefix.endsWith("/") || prefix === "" ? prefix : prefix + "/"; // Create S3 client const clientConfig = { region, }; if (accessKey && secretKey) { clientConfig.credentials = { accessKeyId: accessKey, secretAccessKey: secretKey, }; } else { // Anonymous access for public buckets clientConfig.signer = { sign: async (request) => request }; } if (endpoint) { clientConfig.endpoint = endpoint; clientConfig.forcePathStyle = true; } state.s3Client = new S3Client(clientConfig); // Try listing try { const result = await listS3Prefix(state.s3Prefix); state.corsFallback = false; // Success - show browser show(dom.browserSection); show(dom.browserNormal); hide(dom.browserCorsFallback); dom.s3Status.textContent = "Connected"; dom.s3Status.className = "card-status connected"; show(dom.s3Status); // Build initial tree resetTree(); renderTreeLevel(result, state.s3Prefix, dom.treeContainer, 0); updateStats(); } catch (err) { console.error("S3 listing error:", err); const msg = err.message || String(err); // Check if it's a credentials error vs CORS if (msg.includes("403") || msg.includes("AccessDenied") || msg.includes("InvalidAccessKeyId") || msg.includes("SignatureDoesNotMatch")) { dom.s3ConnectError.textContent = "Invalid credentials or access denied. Check your Access Key and Secret."; dom.s3ConnectError.style.color = "var(--error)"; setLoading(dom.btnConnect, false, "Connect & Browse"); return; } if (msg.includes("404") || msg.includes("NoSuchBucket")) { dom.s3ConnectError.textContent = "Bucket not found. Check the bucket name and region."; dom.s3ConnectError.style.color = "var(--error)"; setLoading(dom.btnConnect, false, "Connect & Browse"); return; } // CORS or network error - fallback mode state.corsFallback = true; show(dom.browserSection); hide(dom.browserNormal); show(dom.browserCorsFallback); dom.s3Status.textContent = "Connected (no browse)"; dom.s3Status.className = "card-status connected"; show(dom.s3Status); } setLoading(dom.btnConnect, false, "Connect & Browse"); } async function listS3Prefix(prefix, continuationToken) { const params = { Bucket: state.s3Bucket, Prefix: prefix || undefined, Delimiter: "/", MaxKeys: 1000, }; if (continuationToken) { params.ContinuationToken = continuationToken; } const command = new ListObjectsV2Command(params); const response = await state.s3Client.send(command); return { folders: (response.CommonPrefixes || []).map((cp) => cp.Prefix), files: (response.Contents || []) .filter((obj) => obj.Key !== prefix) // exclude the prefix itself .map((obj) => ({ key: obj.Key, size: obj.Size || 0, lastModified: obj.LastModified, })), isTruncated: response.IsTruncated, continuationToken: response.NextContinuationToken, }; } // ============================================================ // Tree View // ============================================================ function resetTree() { state.tree = []; state.allFiles.clear(); state.allFolders.clear(); state.totalFiles = 0; state.totalSize = 0; state.selectedFiles = 0; state.selectedSize = 0; dom.treeContainer.innerHTML = ""; } function renderTreeLevel(result, parentPrefix, container, depth) { // Render folders for (const folderKey of result.folders) { const relativeName = folderKey.slice(parentPrefix.length); state.allFolders.set(folderKey, { selected: true, loaded: false }); const node = createTreeNode({ key: folderKey, name: relativeName.replace(/\/$/, ""), isFolder: true, size: 0, depth, }); container.appendChild(node); } // Render files for (const file of result.files) { const relativeName = file.key.slice(parentPrefix.length); state.allFiles.set(file.key, { size: file.size, selected: true }); state.totalFiles++; state.totalSize += file.size; state.selectedFiles++; state.selectedSize += file.size; const node = createTreeNode({ key: file.key, name: relativeName, isFolder: false, size: file.size, depth, }); container.appendChild(node); } // Load more button if (result.isTruncated) { const loadMore = document.createElement("div"); loadMore.className = "tree-load-more"; loadMore.style.paddingLeft = `${(depth + 1) * 24 + 12}px`; loadMore.textContent = "Load more..."; loadMore.addEventListener("click", async () => { loadMore.textContent = "Loading..."; try { const moreResult = await listS3Prefix(parentPrefix, result.continuationToken); loadMore.remove(); renderTreeLevel(moreResult, parentPrefix, container, depth); updateStats(); } catch (err) { loadMore.textContent = `Error: ${err.message}. Click to retry.`; } }); container.appendChild(loadMore); } } function createTreeNode({ key, name, isFolder, size, depth }) { const row = document.createElement("div"); row.className = "tree-node"; row.dataset.key = key; row.dataset.isFolder = isFolder; // Indent const indent = document.createElement("span"); indent.className = "indent"; indent.style.width = `${depth * 24}px`; row.appendChild(indent); // Expand icon (folders only) const expandIcon = document.createElement("span"); expandIcon.className = `expand-icon ${isFolder ? "" : "hidden"}`; expandIcon.textContent = "\u25B6"; // right triangle row.appendChild(expandIcon); // Checkbox const checkbox = document.createElement("input"); checkbox.type = "checkbox"; checkbox.className = "node-checkbox"; checkbox.checked = true; // selected by default checkbox.addEventListener("change", (e) => { e.stopPropagation(); toggleSelection(key, isFolder, checkbox.checked); }); checkbox.addEventListener("click", (e) => e.stopPropagation()); row.appendChild(checkbox); // Icon const icon = document.createElement("span"); icon.className = "node-icon"; icon.textContent = isFolder ? "\uD83D\uDCC1" : "\uD83D\uDCC4"; row.appendChild(icon); // Name const nameSpan = document.createElement("span"); nameSpan.className = "node-name"; nameSpan.textContent = name; row.appendChild(nameSpan); // Meta (size for files) const meta = document.createElement("span"); meta.className = "node-meta"; meta.textContent = isFolder ? "" : formatSize(size); row.appendChild(meta); // Click handler for folders (expand/collapse) if (isFolder) { row.addEventListener("click", () => expandFolder(key, row, depth)); } return row; } async function expandFolder(folderKey, row, depth) { const expandIcon = row.querySelector(".expand-icon"); const folderState = state.allFolders.get(folderKey); // Toggle collapse if (expandIcon.classList.contains("expanded")) { expandIcon.classList.remove("expanded"); // Hide children let next = row.nextElementSibling; while (next && next.dataset.parentFolder === folderKey) { next.style.display = "none"; next = next.nextElementSibling; } return; } expandIcon.classList.add("expanded"); // If already loaded, just show children if (folderState && folderState.loaded) { let next = row.nextElementSibling; while (next && next.dataset.parentFolder === folderKey) { next.style.display = ""; next = next.nextElementSibling; } return; } // Load children const loadingRow = document.createElement("div"); loadingRow.className = "tree-node loading"; loadingRow.dataset.parentFolder = folderKey; loadingRow.innerHTML = ` Loading... `; row.after(loadingRow); try { const result = await listS3Prefix(folderKey); loadingRow.remove(); folderState.loaded = true; // Update meta with count const meta = row.querySelector(".node-meta"); const totalItems = result.folders.length + result.files.length + (result.isTruncated ? "+" : ""); meta.textContent = `${totalItems} items`; // Create a container fragment const fragment = document.createDocumentFragment(); const tempContainer = document.createElement("div"); renderTreeLevel(result, folderKey, tempContainer, depth + 1); // Mark all children with parent folder for collapse/expand for (const child of tempContainer.children) { child.dataset.parentFolder = folderKey; } // Insert after the folder row let insertAfter = row; while (tempContainer.firstChild) { const child = tempContainer.firstChild; child.dataset.parentFolder = folderKey; insertAfter.after(child); insertAfter = child; } // If folder is selected, ensure children are selected too if (folderState.selected) { // Children are already selected by default from renderTreeLevel } updateStats(); } catch (err) { loadingRow.querySelector(".node-name").textContent = `Error: ${err.message}`; loadingRow.querySelector(".node-icon").innerHTML = ""; } } function toggleSelection(key, isFolder, selected) { if (isFolder) { const folderState = state.allFolders.get(key); if (folderState) folderState.selected = selected; // Toggle all loaded children for (const [fileKey, fileState] of state.allFiles) { if (fileKey.startsWith(key)) { const wasSelected = fileState.selected; fileState.selected = selected; if (wasSelected !== selected) { state.selectedFiles += selected ? 1 : -1; state.selectedSize += selected ? fileState.size : -fileState.size; } // Update checkbox in DOM const row = dom.treeContainer.querySelector(`[data-key="${CSS.escape(fileKey)}"]`); if (row) row.querySelector(".node-checkbox").checked = selected; } } // Toggle child folders for (const [folderKeyChild, folderState2] of state.allFolders) { if (folderKeyChild.startsWith(key) && folderKeyChild !== key) { folderState2.selected = selected; const row = dom.treeContainer.querySelector(`[data-key="${CSS.escape(folderKeyChild)}"]`); if (row) row.querySelector(".node-checkbox").checked = selected; } } } else { const fileState = state.allFiles.get(key); if (fileState) { const wasSelected = fileState.selected; fileState.selected = selected; if (wasSelected !== selected) { state.selectedFiles += selected ? 1 : -1; state.selectedSize += selected ? fileState.size : -fileState.size; } } } updateStats(); } function selectAll(selected) { for (const [key, fileState] of state.allFiles) { fileState.selected = selected; } for (const [key, folderState] of state.allFolders) { folderState.selected = selected; } // Update all checkboxes dom.treeContainer.querySelectorAll(".node-checkbox").forEach((cb) => { cb.checked = selected; }); // Recalculate if (selected) { state.selectedFiles = state.totalFiles; state.selectedSize = state.totalSize; } else { state.selectedFiles = 0; state.selectedSize = 0; } updateStats(); } function updateStats() { dom.statSelectedCount.textContent = state.selectedFiles.toLocaleString(); dom.statSelectedSize.textContent = formatSize(state.selectedSize); dom.statTotalCount.textContent = state.totalFiles.toLocaleString(); dom.statTotalSize.textContent = formatSize(state.totalSize); } // ============================================================ // Import Configuration // ============================================================ function showImportSection() { show(dom.importSection); // Default destination bucket to the S3 bucket name if not already set if (!dom.destBucket.value.trim() && state.s3Bucket) { dom.destBucket.value = state.s3Bucket; updateDestPreview(); } dom.importSection.scrollIntoView({ behavior: "smooth", block: "start" }); // Update patterns preview updatePatternsPreview(); } function updateDestPreview() { const bucket = dom.destBucket.value.trim(); if (!bucket) { hide(dom.destPreview); return; } const fullBucket = bucket.includes("/") ? bucket : `${state.username}/${bucket}`; const prefix = dom.destPrefix.value.trim(); const path = prefix ? `${fullBucket}/${prefix.replace(/\/$/, "")}` : fullBucket; dom.destPreview.textContent = `hf://buckets/${path}`; show(dom.destPreview); } function computePatterns() { if (state.corsFallback) { const includes = dom.manualInclude.value.trim().split("\n").filter(Boolean); const excludes = dom.manualExclude.value.trim().split("\n").filter(Boolean); return { include: includes, exclude: excludes }; } // All files selected? if (state.selectedFiles === state.totalFiles) { return { include: [], exclude: [] }; } // No files selected? if (state.selectedFiles === 0) { return { include: ["__nothing__"], exclude: [] }; } // Compute selected/deselected const selected = []; const deselected = []; for (const [key, fileState] of state.allFiles) { const relKey = key.startsWith(state.s3Prefix) ? key.slice(state.s3Prefix.length) : key; if (fileState.selected) { selected.push(relKey); } else { deselected.push(relKey); } } // Use whichever set is smaller if (deselected.length <= selected.length) { return { include: [], exclude: deselected }; } else { // Try to group by folder const patterns = compactPatterns(selected); return { include: patterns, exclude: [] }; } } function compactPatterns(keys) { // Group files by their parent folder and check if entire folder is selected const folderCounts = new Map(); const folderTotals = new Map(); for (const key of keys) { const parts = key.split("/"); if (parts.length > 1) { const folder = parts.slice(0, -1).join("/") + "/"; folderCounts.set(folder, (folderCounts.get(folder) || 0) + 1); } } // Count total files per folder for (const [fileKey] of state.allFiles) { const relKey = fileKey.startsWith(state.s3Prefix) ? fileKey.slice(state.s3Prefix.length) : fileKey; const parts = relKey.split("/"); if (parts.length > 1) { const folder = parts.slice(0, -1).join("/") + "/"; folderTotals.set(folder, (folderTotals.get(folder) || 0) + 1); } } const patterns = []; const coveredKeys = new Set(); // If entire folder is selected, use folder/* for (const [folder, count] of folderCounts) { if (count === (folderTotals.get(folder) || 0)) { patterns.push(`${folder}*`); for (const key of keys) { if (key.startsWith(folder)) coveredKeys.add(key); } } } // Add remaining individual files for (const key of keys) { if (!coveredKeys.has(key)) { patterns.push(key); } } return patterns; } function updatePatternsPreview() { const patterns = computePatterns(); const s3Uri = `s3://${state.s3Bucket}/${state.s3Prefix}`; const destBucket = dom.destBucket.value.trim(); const nsDest = destBucket.includes("/") ? destBucket : `${state.username}/${destBucket}`; const destPrefix = dom.destPrefix.value.trim(); const dest = destPrefix ? `hf://buckets/${nsDest}/${destPrefix.replace(/\/$/, "")}` : `hf://buckets/${nsDest}`; const workers = dom.jobWorkers.value || "8"; let cmd = `hf buckets import ${shellEscape(s3Uri)} ${shellEscape(dest)}`; if (patterns.include.length > 0) { for (const p of patterns.include) { cmd += ` --include ${shellEscape(p)}`; } } if (patterns.exclude.length > 0) { for (const p of patterns.exclude) { cmd += ` --exclude ${shellEscape(p)}`; } } cmd += ` --workers ${workers}`; cmd += ` --buffer-size 49g`; dom.patternsPreview.textContent = cmd; show(dom.patternsInfo); } // ============================================================ // Bucket Creation // ============================================================ async function ensureBucketExists(namespace, bucketName, isPrivate) { const token = state.oauthResult.accessToken; // Use "me" for the current user's namespace, matching the Python SDK behavior const ns = namespace === state.username ? "me" : namespace; const resp = await fetch(`https://huggingface.co/api/buckets/${ns}/${bucketName}`, { method: "POST", headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", }, body: JSON.stringify({ private: isPrivate }), }); if (resp.ok || resp.status === 409) { // Created or already exists return true; } const errorText = await resp.text(); throw new Error(`Failed to create bucket: ${resp.status} - ${errorText}`); } // ============================================================ // Job Submission // ============================================================ async function submitImportJob() { const destBucketInput = dom.destBucket.value.trim(); if (!destBucketInput) { showImportError("Please provide a destination bucket name."); return; } // Parse namespace/bucket let namespace, bucketName; if (destBucketInput.includes("/")) { [namespace, bucketName] = destBucketInput.split("/", 2); } else { namespace = state.username; bucketName = destBucketInput; } const destPrefix = dom.destPrefix.value.trim().replace(/\/$/, ""); const fullDest = destPrefix ? `hf://buckets/${namespace}/${bucketName}/${destPrefix}` : `hf://buckets/${namespace}/${bucketName}`; const s3Uri = `s3://${state.s3Bucket}/${state.s3Prefix}`; const flavor = dom.jobFlavor.value; const workers = parseInt(dom.jobWorkers.value) || 8; const timeout = parseInt(dom.jobTimeout.value) || 7200; const token = state.oauthResult.accessToken; hideImportError(); setLoading(dom.btnStartImport, true, "Creating bucket & submitting job..."); try { // Step 1: Ensure bucket exists const isPrivate = dom.destPrivate.checked; await ensureBucketExists(namespace, bucketName, isPrivate); // Step 2: Build command const patterns = computePatterns(); let script = "pip install -q 'huggingface_hub[s3] @ git+https://github.com/huggingface/huggingface_hub.git@cursor/s3-to-hf-bucket-ingestion-f144'"; script += ` && hf buckets import ${shellEscape(s3Uri)} ${shellEscape(fullDest)}`; if (patterns.include.length > 0) { for (const p of patterns.include) { script += ` --include ${shellEscape(p)}`; } } if (patterns.exclude.length > 0) { for (const p of patterns.exclude) { script += ` --exclude ${shellEscape(p)}`; } } script += ` --workers ${workers}`; script += ` --buffer-size 49g`; // Step 3: Submit job const secrets = { HF_TOKEN: token }; if (state.awsAccessKeyId) secrets.AWS_ACCESS_KEY_ID = state.awsAccessKeyId; if (state.awsSecretAccessKey) secrets.AWS_SECRET_ACCESS_KEY = state.awsSecretAccessKey; const jobPayload = { dockerImage: "python:3.12", command: ["bash", "-c", script], arguments: [], flavor, timeout, environment: { AWS_DEFAULT_REGION: state.s3Region || "us-east-1", }, secrets, }; if (state.s3Endpoint) { jobPayload.environment.AWS_ENDPOINT_URL = state.s3Endpoint; } const resp = await fetch(`https://huggingface.co/api/jobs/${namespace}`, { method: "POST", headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", }, body: JSON.stringify(jobPayload), }); if (!resp.ok) { const errorText = await resp.text(); throw new Error(`Job creation failed (${resp.status}): ${errorText}`); } const jobData = await resp.json(); state.jobId = jobData.id; state.jobNamespace = namespace; // Show status showJobStatus(jobData); pollJobStatus(); } catch (err) { showImportError(err.message); setLoading(dom.btnStartImport, false, "Start Import"); } } function showImportError(msg) { dom.importError.textContent = msg; show(dom.importError); } function hideImportError() { hide(dom.importError); } // ============================================================ // Job Status // ============================================================ function showJobStatus(jobData) { hide(dom.s3Section); hide(dom.browserSection); hide(dom.importSection); show(dom.statusSection); const jobUrl = `https://huggingface.co/jobs/${state.jobNamespace}/${jobData.id}`; dom.jobIdDisplay.textContent = `Job ID: ${jobData.id}`; dom.jobLink.href = jobUrl; // Build bucket URL from destination const destBucketInput = dom.destBucket.value.trim(); let bucketNs, bucketName; if (destBucketInput.includes("/")) { [bucketNs, bucketName] = destBucketInput.split("/", 2); } else { bucketNs = state.username; bucketName = destBucketInput; } dom.bucketLink.href = `https://huggingface.co/buckets/${bucketNs}/${bucketName}`; updateJobStatusUI(jobData.status?.stage || "SCHEDULING"); } function updateJobStatusUI(stage) { const lowerStage = stage.toLowerCase(); dom.jobStatusBadge.textContent = stage; dom.jobStatusBadge.className = `job-status-badge ${lowerStage}`; switch (stage) { case "SCHEDULING": dom.jobStatusIcon.textContent = "\uD83D\uDE80"; dom.jobStatusTitle.textContent = "Import Job Submitted"; dom.jobStatusMessage.textContent = "Your job is being scheduled. This usually takes a few seconds."; break; case "RUNNING": dom.jobStatusIcon.textContent = "\u2699\uFE0F"; dom.jobStatusTitle.textContent = "Import in Progress"; dom.jobStatusMessage.textContent = "Your data is being transferred from S3 to your HF bucket. You can close this page - the job will continue running."; break; case "COMPLETED": dom.jobStatusIcon.textContent = "\u2705"; dom.jobStatusTitle.textContent = "Import Complete!"; dom.jobStatusMessage.textContent = "Your data has been successfully imported to the HF bucket."; break; case "ERROR": dom.jobStatusIcon.textContent = "\u274C"; dom.jobStatusTitle.textContent = "Import Failed"; dom.jobStatusMessage.textContent = "The import job encountered an error. Check the job logs for details."; break; case "CANCELED": dom.jobStatusIcon.textContent = "\u23F9\uFE0F"; dom.jobStatusTitle.textContent = "Import Canceled"; dom.jobStatusMessage.textContent = "The import job was canceled."; break; default: dom.jobStatusIcon.textContent = "\u2753"; dom.jobStatusTitle.textContent = `Status: ${stage}`; dom.jobStatusMessage.textContent = ""; } } async function pollJobStatus() { if (!state.jobId) return; const token = state.oauthResult.accessToken; const terminalStages = new Set(["COMPLETED", "ERROR", "CANCELED", "DELETED"]); const poll = async () => { try { const resp = await fetch( `https://huggingface.co/api/jobs/${state.jobNamespace}/${state.jobId}`, { headers: { Authorization: `Bearer ${token}` } } ); if (resp.ok) { const data = await resp.json(); const stage = data.status?.stage || "UNKNOWN"; updateJobStatusUI(stage); if (terminalStages.has(stage)) return; // stop polling } } catch { // Network error, keep polling } setTimeout(poll, 5000); }; // Start polling after a short delay setTimeout(poll, 3000); } function resetForNewImport() { hide(dom.statusSection); show(dom.s3Section); state.jobId = null; state.jobNamespace = null; // Reset browser and import sections hide(dom.browserSection); hide(dom.importSection); hide(dom.s3Status); setLoading(dom.btnStartImport, false, "Start Import"); } // ============================================================ // Event Listeners // ============================================================ function initEventListeners() { dom.btnConnect.addEventListener("click", connectS3); dom.btnSelectAll.addEventListener("click", () => selectAll(true)); dom.btnDeselectAll.addEventListener("click", () => selectAll(false)); dom.btnContinueImport.addEventListener("click", showImportSection); dom.destBucket.addEventListener("input", () => { updateDestPreview(); updatePatternsPreview(); }); dom.destPrefix.addEventListener("input", () => { updateDestPreview(); updatePatternsPreview(); }); dom.jobFlavor.addEventListener("change", () => { updateWorkersFromHardware(); updatePatternsPreview(); }); dom.jobWorkers.addEventListener("input", updatePatternsPreview); dom.btnStartImport.addEventListener("click", submitImportJob); dom.btnNewImport.addEventListener("click", resetForNewImport); // Allow Enter key on S3 form dom.s3Section.addEventListener("keydown", (e) => { if (e.key === "Enter") { e.preventDefault(); connectS3(); } }); // Help toggle buttons document.addEventListener("click", (e) => { const toggle = e.target.closest(".help-toggle"); if (!toggle) return; e.preventDefault(); const helpId = toggle.dataset.help; if (!helpId) return; const helpEl = document.getElementById(helpId); if (!helpEl) return; helpEl.classList.toggle("visible"); }); } // ============================================================ // Init // ============================================================ async function main() { initEventListeners(); await initOAuth(); } main().catch((err) => { console.error("App initialization error:", err); });