From c74898049a66ef5eaa3140658b99623c6390c189 Mon Sep 17 00:00:00 2001 From: MayaTheShy Date: Thu, 26 Mar 2026 13:10:49 -0400 Subject: [PATCH] feat: add download-textures script for bulk downloading Minecraft and Create textures --- web/server/download-textures.js | 173 ++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) create mode 100644 web/server/download-textures.js diff --git a/web/server/download-textures.js b/web/server/download-textures.js new file mode 100644 index 0000000..83ec91c --- /dev/null +++ b/web/server/download-textures.js @@ -0,0 +1,173 @@ +#!/usr/bin/env node +/** + * download-textures.js — Bulk-download all Minecraft, Create & CC:Tweaked + * item/block textures into the local texture cache. + * + * Run once (or on container start) to pre-populate the cache so the proxy + * never needs to hit upstream for known textures. + * + * Usage: node download-textures.js [cacheDir] + * Default: /data/texture-cache (matches server.js) + */ + +import fs from 'fs'; +import path from 'path'; + +const CACHE_DIR = process.argv[2] || process.env.TEXTURE_CACHE_DIR || '/data/texture-cache'; + +// ── Upstream repos (same as server.js) ───────────────────────────────────── + +const REPOS = { + minecraft: { + api: 'https://api.github.com/repos/InventivetalentDev/minecraft-assets/git/trees/1.21.4?recursive=1', + raw: 'https://cdn.jsdelivr.net/gh/InventivetalentDev/minecraft-assets@1.21.4', + prefix: 'assets/minecraft/textures/', + folders: ['item/', 'block/'], // only grab these sub-folders + }, + create: { + api: 'https://api.github.com/repos/Creators-of-Create/Create/git/trees/mc1.20.1/dev?recursive=1', + raw: 'https://raw.githubusercontent.com/Creators-of-Create/Create/mc1.20.1/dev', + prefix: 'src/main/resources/assets/create/textures/', + folders: ['item/', 'block/'], + }, + computercraft: { + api: 'https://api.github.com/repos/cc-tweaked/CC-Tweaked/git/trees/mc-1.20.x?recursive=1', + raw: 'https://raw.githubusercontent.com/cc-tweaked/CC-Tweaked/mc-1.20.x', + prefix: 'projects/common/src/main/resources/assets/computercraft/textures/', + folders: ['item/', 'block/'], + }, +}; + +// ── Rate-limit-friendly parallel downloader ──────────────────────────────── + +const CONCURRENCY = 10; +const RETRY_DELAY = 1000; +const MAX_RETRIES = 3; + +async function downloadFile(url, dest, retries = 0) { + try { + const res = await fetch(url); + if (res.status === 404) return false; // genuinely missing + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const buffer = Buffer.from(await res.arrayBuffer()); + fs.mkdirSync(path.dirname(dest), { recursive: true }); + fs.writeFileSync(dest, buffer); + return true; + } catch (err) { + if (retries < MAX_RETRIES) { + await new Promise(r => setTimeout(r, RETRY_DELAY * (retries + 1))); + return downloadFile(url, dest, retries + 1); + } + console.error(` ✗ ${url}: ${err.message}`); + return false; + } +} + +async function runPool(tasks) { + let idx = 0; + let ok = 0; + let fail = 0; + async function worker() { + while (idx < tasks.length) { + const task = tasks[idx++]; + const success = await downloadFile(task.url, task.dest); + if (success) ok++; + else fail++; + } + } + const workers = Array.from({ length: Math.min(CONCURRENCY, tasks.length) }, () => worker()); + await Promise.all(workers); + return { ok, fail }; +} + +// ── Main ─────────────────────────────────────────────────────────────────── + +async function downloadNamespace(name, repo) { + console.log(`\n⬇ ${name}: fetching file list…`); + + const res = await fetch(repo.api, { + headers: { 'User-Agent': 'inventory-manager-texture-dl' }, + }); + if (!res.ok) { + console.error(` ✗ GitHub API ${res.status}: ${await res.text()}`); + return; + } + const data = await res.json(); + + // Filter to only PNG files in the target folders under the prefix + const tasks = []; + for (const entry of data.tree || []) { + if (entry.type !== 'blob') continue; + if (!entry.path.startsWith(repo.prefix)) continue; + if (!entry.path.endsWith('.png')) continue; + + const relPath = entry.path.slice(repo.prefix.length); // e.g. "item/diamond.png" + const inFolder = repo.folders.some(f => relPath.startsWith(f)); + if (!inFolder) continue; + + const dest = path.join(CACHE_DIR, name, relPath); + if (fs.existsSync(dest)) continue; // already cached + + tasks.push({ + url: `${repo.raw}/${entry.path}`, + dest, + }); + } + + if (tasks.length === 0) { + console.log(` ✓ ${name}: all textures already cached`); + return; + } + + console.log(` ↓ downloading ${tasks.length} textures…`); + const { ok, fail } = await runPool(tasks); + console.log(` ✓ ${name}: ${ok} downloaded, ${fail} failed`); +} + +async function main() { + console.log(`Texture cache dir: ${CACHE_DIR}`); + fs.mkdirSync(CACHE_DIR, { recursive: true }); + + for (const [name, repo] of Object.entries(REPOS)) { + await downloadNamespace(name, repo); + } + + // Clean up any .miss files so newly-downloaded textures take effect + let cleared = 0; + function cleanMiss(dir) { + try { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) cleanMiss(full); + else if (entry.name.endsWith('.miss')) { fs.unlinkSync(full); cleared++; } + } + } catch (_) { /* ignore */ } + } + cleanMiss(CACHE_DIR); + if (cleared) console.log(`\n🗑 Cleared ${cleared} negative-cache entries`); + + // Build and print index stats + const stats = {}; + function countPngs(dir, ns) { + try { + for (const entry of fs.readdirSync(dir, { withFileTypes: true })) { + const full = path.join(dir, entry.name); + if (entry.isDirectory()) countPngs(full, ns); + else if (entry.name.endsWith('.png')) stats[ns] = (stats[ns] || 0) + 1; + } + } catch (_) { /* ignore */ } + } + for (const ns of Object.keys(REPOS)) { + countPngs(path.join(CACHE_DIR, ns), ns); + } + console.log('\n📊 Cache totals:'); + for (const [ns, count] of Object.entries(stats)) { + console.log(` ${ns}: ${count} textures`); + } + console.log('\n✅ Done'); +} + +main().catch(err => { + console.error('Fatal:', err); + process.exit(1); +});