Optimize import: bulk INSERTs, pre-resolve rarities, BATCH_SIZE 50->500

src/controllers/importController.js

@@ -1,33 +1,28 @@
-const { upsertCard } = require('../models/cardModel');
-const { upsertRarity } = require('../models/rarityModel');
-const { upsertSet } = require('../models/setModel');
-const { insertCardSetRarity } = require('../models/cardSetRarityModel');
-const { insertCardImage } = require('../models/cardImageModel');
 const { getLocalDBVersion, setLocalDBVersion } = require('../models/dbVersionModel');
 const { fetchAllCards, fetchAllSets, fetchDatabaseVersion } = require('../services/ygoproService');
 const db = require('../config/db');
 
-const BATCH_SIZE = 50;
+const BATCH_SIZE = 500;
 
 async function importSetsInternal() {
   const sets = await fetchAllSets();
-  let added = 0;
+  if (!sets.length) return { added: 0, total: 0 };
 
-  for (let i = 0; i < sets.length; i += BATCH_SIZE) {
+  const values = sets.map(() => '(?, ?, ?, ?)').join(', ');
-    const batch = sets.slice(i, i + BATCH_SIZE);
+  const params = sets.flatMap(s => [s.set_name, s.set_code, s.num_of_cards, s.tcg_date]);
-    await Promise.all(batch.map(async (set) => {
-      await upsertSet(set);
-      added++;
-    }));
-  }
 
-  return {
+  await db.execute(`
-    added,
+    INSERT INTO sets (set_name, set_code, num_of_cards, tcg_date)
-    total: sets.length
+    VALUES ${values}
-  };
+    ON DUPLICATE KEY UPDATE
+      set_name = VALUES(set_name),
+      num_of_cards = VALUES(num_of_cards),
+      tcg_date = VALUES(tcg_date)
+  `, params);
+
+  return { added: sets.length, total: sets.length };
 }
 
-// Express handler for /sets endpoint
 async function importSets(req, res) {
   try {
     const result = await importSetsInternal();
@@ -43,114 +38,123 @@ async function importCardsInternal() {
   const cards = await fetchAllCards();
   const totalCards = cards.length;
 
-  let addedCards = 0, addedImages = 0, addedRarities = 0;
+  // Pre-load set and rarity caches
-
-  const rarityCache = {};
-  const setCache = {};
-
   const [setsRows] = await db.execute('SELECT id, set_name FROM sets');
-  for (const row of setsRows) {
+  const setCache = Object.fromEntries(setsRows.map(r => [r.set_name, r.id]));
-    setCache[row.set_name] = row.id;
-  }
 
   const [rarityRows] = await db.execute('SELECT id, rarity_name FROM rarities');
-  for (const row of rarityRows) {
+  const rarityCache = Object.fromEntries(rarityRows.map(r => [r.rarity_name, r.id]));
-    rarityCache[row.rarity_name] = row.id;
+
+  // Collect all unique rarities from the full card list and upsert them before the main loop
+  const unseenRarities = new Map();
+  for (const card of cards) {
+    if (!Array.isArray(card.card_sets)) continue;
+    for (const set of card.card_sets) {
+      if (set.set_rarity && !rarityCache[set.set_rarity] && !unseenRarities.has(set.set_rarity)) {
+        unseenRarities.set(set.set_rarity, set.set_rarity_code);
+      }
+    }
   }
 
+  if (unseenRarities.size) {
+    const rarityValues = Array.from(unseenRarities.keys()).map(() => '(?, ?)').join(', ');
+    const rarityParams = Array.from(unseenRarities.entries()).flatMap(([name, code]) => [name, code]);
+    await db.execute(`
+      INSERT INTO rarities (rarity_name, rarity_code)
+      VALUES ${rarityValues}
+      ON DUPLICATE KEY UPDATE rarity_code = VALUES(rarity_code)
+    `, rarityParams);
+
+    const [newRarityRows] = await db.execute('SELECT id, rarity_name FROM rarities');
+    for (const row of newRarityRows) rarityCache[row.rarity_name] = row.id;
+  }
+
+  let addedCards = 0, addedImages = 0, addedSetRarities = 0;
+
   for (let i = 0; i < totalCards; i += BATCH_SIZE) {
     const batch = cards.slice(i, i + BATCH_SIZE);
+
+    const cardValues = [], cardParams = [];
+    const imageValues = [], imageParams = [];
+    const csrValues = [], csrParams = [];
+
+    for (const card of batch) {
+      cardValues.push('(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
+      cardParams.push(
+        card.id, card.name, card.type, card.frameType,
+        card.level || null, card.race || null, card.attribute || null,
+        card.linkval || null, card.tcg_date || null, card.ocg_date || null
+      );
+
+      if (Array.isArray(card.card_images)) {
+        for (const img of card.card_images) {
+          imageValues.push('(?, ?, ?)');
+          imageParams.push(img.id, card.id, img.image_url);
+        }
+      }
+
+      if (Array.isArray(card.card_sets)) {
+        for (const set of card.card_sets) {
+          const set_id = setCache[set.set_name];
+          const rarity_id = rarityCache[set.set_rarity];
+          if (set_id && rarity_id) {
+            csrValues.push('(?, ?, ?, ?)');
+            csrParams.push(card.id, set_id, rarity_id, set.set_code);
+          }
+        }
+      }
+    }
+
     const conn = await db.getConnection();
     try {
       await conn.beginTransaction();
 
-      await Promise.all(batch.map(async (card) => {
+      await conn.execute(`
+        INSERT INTO cards (id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date)
+        VALUES ${cardValues.join(', ')}
+        ON DUPLICATE KEY UPDATE
+          name = VALUES(name), card_type = VALUES(card_type), frame_type = VALUES(frame_type),
+          level = VALUES(level), race = VALUES(race), attribute = VALUES(attribute),
+          link_val = VALUES(link_val), tcg_date = VALUES(tcg_date), ocg_date = VALUES(ocg_date)
+      `, cardParams);
+
+      if (imageValues.length) {
         await conn.execute(`
-          INSERT INTO cards
+          INSERT INTO card_images (image_id, card_id, image_url)
-            (id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date)
+          VALUES ${imageValues.join(', ')}
-          VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+          ON DUPLICATE KEY UPDATE image_url = VALUES(image_url)
-          ON DUPLICATE KEY UPDATE
+        `, imageParams);
-            name = VALUES(name),
+      }
-            card_type = VALUES(card_type),
-            frame_type = VALUES(frame_type),
-            level = VALUES(level),
-            race = VALUES(race),
-            attribute = VALUES(attribute),
-            link_val = VALUES(link_val),
-            tcg_date = VALUES(tcg_date),
-            ocg_date = VALUES(ocg_date)
-        `, [
-          card.id,
-          card.name,
-          card.type,
-          card.frameType,
-          card.level || null,
-          card.race || null,
-          card.attribute || null,
-          card.linkval || null,
-          card.tcg_date || null,
-          card.ocg_date || null
-        ]);
-        addedCards++;
 
-        if (Array.isArray(card.card_sets)) {
+      if (csrValues.length) {
-          for (const set of card.card_sets) {
+        await conn.execute(`
-            if (!rarityCache[set.set_rarity]) {
+          INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code)
-              await conn.execute(`
+          VALUES ${csrValues.join(', ')}
-                INSERT INTO rarities (rarity_name, rarity_code)
+          ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code)
-                VALUES (?, ?)
+        `, csrParams);
-                ON DUPLICATE KEY UPDATE rarity_code = VALUES(rarity_code)
+      }
-              `, [set.set_rarity, set.set_rarity_code]);
-              const [rows] = await conn.execute('SELECT id FROM rarities WHERE rarity_name = ? LIMIT 1', [set.set_rarity]);
-              rarityCache[set.set_rarity] = rows[0].id;
-              addedRarities++;
-            }
-            const rarity_id = rarityCache[set.set_rarity];
-            const set_id = setCache[set.set_name];
-            if (set_id) {
-              await conn.execute(`
-                INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code)
-                VALUES (?, ?, ?, ?)
-                ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code)
-              `, [card.id, set_id, rarity_id, set.set_code]);
-            }
-          }
-        }
-
-        if (Array.isArray(card.card_images)) {
-          for (const img of card.card_images) {
-            await conn.execute(`
-              INSERT INTO card_images (image_id, card_id, image_url)
-              VALUES (?, ?, ?)
-              ON DUPLICATE KEY UPDATE image_url = VALUES(image_url)
-            `, [img.id, card.id, img.image_url]);
-            addedImages++;
-          }
-        }
-      }));
 
       await conn.commit();
+      addedCards += batch.length;
+      addedImages += imageValues.length;
+      addedSetRarities += csrValues.length;
     } catch (err) {
       await conn.rollback();
       throw err;
     } finally {
       conn.release();
     }
-
   }
 
-  const durationSeconds = ((Date.now() - startTime) / 1000).toFixed(2);
-
   return {
     total_cards: totalCards,
     cards_added: addedCards,
     images_added: addedImages,
-    rarities_added: addedRarities,
+    set_rarities_added: addedSetRarities,
-    duration_seconds: parseFloat(durationSeconds)
+    duration_seconds: parseFloat(((Date.now() - startTime) / 1000).toFixed(2)),
   };
 }
 
-// Express handler for /cards
 async function importCards(req, res) {
   try {
     const result = await importCardsInternal();
@@ -161,7 +165,6 @@ async function importCards(req, res) {
   }
 }
 
-
 async function importFullDatabase(req, res) {
   const startTime = Date.now();
   try {