feat: add download-textures script for bulk downloading Minecraft and Create textures

web/server/download-textures.js

@@ -0,0 +1,173 @@
+#!/usr/bin/env node
+/**
+ * download-textures.js — Bulk-download all Minecraft, Create & CC:Tweaked
+ * item/block textures into the local texture cache.
+ *
+ * Run once (or on container start) to pre-populate the cache so the proxy
+ * never needs to hit upstream for known textures.
+ *
+ * Usage:   node download-textures.js [cacheDir]
+ * Default: /data/texture-cache  (matches server.js)
+ */
+
+import fs from 'fs';
+import path from 'path';
+
+const CACHE_DIR = process.argv[2] || process.env.TEXTURE_CACHE_DIR || '/data/texture-cache';
+
+// ── Upstream repos (same as server.js) ─────────────────────────────────────
+
+const REPOS = {
+  minecraft: {
+    api: 'https://api.github.com/repos/InventivetalentDev/minecraft-assets/git/trees/1.21.4?recursive=1',
+    raw: 'https://cdn.jsdelivr.net/gh/InventivetalentDev/minecraft-assets@1.21.4',
+    prefix: 'assets/minecraft/textures/',
+    folders: ['item/', 'block/'],            // only grab these sub-folders
+  },
+  create: {
+    api: 'https://api.github.com/repos/Creators-of-Create/Create/git/trees/mc1.20.1/dev?recursive=1',
+    raw: 'https://raw.githubusercontent.com/Creators-of-Create/Create/mc1.20.1/dev',
+    prefix: 'src/main/resources/assets/create/textures/',
+    folders: ['item/', 'block/'],
+  },
+  computercraft: {
+    api: 'https://api.github.com/repos/cc-tweaked/CC-Tweaked/git/trees/mc-1.20.x?recursive=1',
+    raw: 'https://raw.githubusercontent.com/cc-tweaked/CC-Tweaked/mc-1.20.x',
+    prefix: 'projects/common/src/main/resources/assets/computercraft/textures/',
+    folders: ['item/', 'block/'],
+  },
+};
+
+// ── Rate-limit-friendly parallel downloader ────────────────────────────────
+
+const CONCURRENCY = 10;
+const RETRY_DELAY = 1000;
+const MAX_RETRIES = 3;
+
+async function downloadFile(url, dest, retries = 0) {
+  try {
+    const res = await fetch(url);
+    if (res.status === 404) return false;          // genuinely missing
+    if (!res.ok) throw new Error(`HTTP ${res.status}`);
+    const buffer = Buffer.from(await res.arrayBuffer());
+    fs.mkdirSync(path.dirname(dest), { recursive: true });
+    fs.writeFileSync(dest, buffer);
+    return true;
+  } catch (err) {
+    if (retries < MAX_RETRIES) {
+      await new Promise(r => setTimeout(r, RETRY_DELAY * (retries + 1)));
+      return downloadFile(url, dest, retries + 1);
+    }
+    console.error(`  ✗ ${url}: ${err.message}`);
+    return false;
+  }
+}
+
+async function runPool(tasks) {
+  let idx = 0;
+  let ok = 0;
+  let fail = 0;
+  async function worker() {
+    while (idx < tasks.length) {
+      const task = tasks[idx++];
+      const success = await downloadFile(task.url, task.dest);
+      if (success) ok++;
+      else fail++;
+    }
+  }
+  const workers = Array.from({ length: Math.min(CONCURRENCY, tasks.length) }, () => worker());
+  await Promise.all(workers);
+  return { ok, fail };
+}
+
+// ── Main ───────────────────────────────────────────────────────────────────
+
+async function downloadNamespace(name, repo) {
+  console.log(`\n⬇  ${name}: fetching file list…`);
+
+  const res = await fetch(repo.api, {
+    headers: { 'User-Agent': 'inventory-manager-texture-dl' },
+  });
+  if (!res.ok) {
+    console.error(`  ✗ GitHub API ${res.status}: ${await res.text()}`);
+    return;
+  }
+  const data = await res.json();
+
+  // Filter to only PNG files in the target folders under the prefix
+  const tasks = [];
+  for (const entry of data.tree || []) {
+    if (entry.type !== 'blob') continue;
+    if (!entry.path.startsWith(repo.prefix)) continue;
+    if (!entry.path.endsWith('.png')) continue;
+
+    const relPath = entry.path.slice(repo.prefix.length);   // e.g. "item/diamond.png"
+    const inFolder = repo.folders.some(f => relPath.startsWith(f));
+    if (!inFolder) continue;
+
+    const dest = path.join(CACHE_DIR, name, relPath);
+    if (fs.existsSync(dest)) continue;                      // already cached
+
+    tasks.push({
+      url: `${repo.raw}/${entry.path}`,
+      dest,
+    });
+  }
+
+  if (tasks.length === 0) {
+    console.log(`  ✓ ${name}: all textures already cached`);
+    return;
+  }
+
+  console.log(`  ↓ downloading ${tasks.length} textures…`);
+  const { ok, fail } = await runPool(tasks);
+  console.log(`  ✓ ${name}: ${ok} downloaded, ${fail} failed`);
+}
+
+async function main() {
+  console.log(`Texture cache dir: ${CACHE_DIR}`);
+  fs.mkdirSync(CACHE_DIR, { recursive: true });
+
+  for (const [name, repo] of Object.entries(REPOS)) {
+    await downloadNamespace(name, repo);
+  }
+
+  // Clean up any .miss files so newly-downloaded textures take effect
+  let cleared = 0;
+  function cleanMiss(dir) {
+    try {
+      for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+        const full = path.join(dir, entry.name);
+        if (entry.isDirectory()) cleanMiss(full);
+        else if (entry.name.endsWith('.miss')) { fs.unlinkSync(full); cleared++; }
+      }
+    } catch (_) { /* ignore */ }
+  }
+  cleanMiss(CACHE_DIR);
+  if (cleared) console.log(`\n🗑  Cleared ${cleared} negative-cache entries`);
+
+  // Build and print index stats
+  const stats = {};
+  function countPngs(dir, ns) {
+    try {
+      for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+        const full = path.join(dir, entry.name);
+        if (entry.isDirectory()) countPngs(full, ns);
+        else if (entry.name.endsWith('.png')) stats[ns] = (stats[ns] || 0) + 1;
+      }
+    } catch (_) { /* ignore */ }
+  }
+  for (const ns of Object.keys(REPOS)) {
+    countPngs(path.join(CACHE_DIR, ns), ns);
+  }
+  console.log('\n📊 Cache totals:');
+  for (const [ns, count] of Object.entries(stats)) {
+    console.log(`   ${ns}: ${count} textures`);
+  }
+  console.log('\n✅ Done');
+}
+
+main().catch(err => {
+  console.error('Fatal:', err);
+  process.exit(1);
+});