Card Import Works

src/controllers/importController.js

@@ -1,7 +1,16 @@
-const { fetchAllSets } = require('../services/ygoproService');
-const { upsertSet } = require('../models/setModel');
+const {
+  upsertCard
+} = require('../models/cardModel');
+const {
+  upsertRarity,
+  getRarityId
+} = require('../models/rarityModel');
+const { insertCardSetRarity } = require('../models/cardSetRarityModel');
+const { insertCardImage } = require('../models/cardImageModel');
+const { fetchAllCards } = require('../services/ygoproService');
+const db = require('../config/db');
 
-const BATCH_SIZE = 10;
+const BATCH_SIZE = 50;
 
 async function importSets(req, res) {
   try {
@@ -27,4 +36,137 @@ async function importSets(req, res) {
   }
 }
 
-module.exports = { importSets };
+async function importCards(req, res) {
+  const startTime = Date.now();
+
+  try {
+    const cards = await fetchAllCards();
+    const totalCards = cards.length;
+
+    let addedCards = 0, addedImages = 0, addedRarities = 0;
+
+    // Cache sets and rarities to reduce repeated DB lookups
+    const rarityCache = {};
+    const setCache = {};
+
+    // Preload sets into cache
+    const [setsRows] = await db.execute('SELECT id, set_code FROM sets');
+    for (const row of setsRows) {
+      setCache[row.set_code] = row.id;
+    }
+
+    // Preload rarities into cache
+    const [rarityRows] = await db.execute('SELECT id, rarity_name FROM rarities');
+    for (const row of rarityRows) {
+      rarityCache[row.rarity_name] = row.id;
+    }
+
+    // Process cards in batches
+    for (let i = 0; i < totalCards; i += BATCH_SIZE) {
+      const batch = cards.slice(i, i + BATCH_SIZE);
+
+      // Wrap batch in a transaction
+      const conn = await db.getConnection();
+      try {
+        await conn.beginTransaction();
+
+        await Promise.all(batch.map(async (card) => {
+          // Upsert card
+          await conn.execute(`
+            INSERT INTO cards
+              (id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date)
+            VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            ON DUPLICATE KEY UPDATE
+              name = VALUES(name),
+              card_type = VALUES(card_type),
+              frame_type = VALUES(frame_type),
+              level = VALUES(level),
+              race = VALUES(race),
+              attribute = VALUES(attribute),
+              link_val = VALUES(link_val),
+              tcg_date = VALUES(tcg_date),
+              ocg_date = VALUES(ocg_date)
+          `, [
+            card.id,
+            card.name,
+            card.type,
+            card.frameType,
+            card.level || null,
+            card.race || null,
+            card.attribute || null,
+            card.linkval || null,
+            card.tcg_date || null,
+            card.ocg_date || null
+          ]);
+          addedCards++;
+
+          // Process rarities and card_sets_rarity
+          if (Array.isArray(card.card_sets)) {
+            for (const set of card.card_sets) {
+              // Upsert rarity if not cached
+              if (!rarityCache[set.set_rarity]) {
+                await conn.execute(`
+                  INSERT INTO rarities (rarity_name, rarity_code)
+                  VALUES (?, ?)
+                  ON DUPLICATE KEY UPDATE rarity_code = VALUES(rarity_code)
+                `, [set.set_rarity, set.set_rarity_code]);
+                const [rows] = await conn.execute('SELECT id FROM rarities WHERE rarity_name = ? LIMIT 1', [set.set_rarity]);
+                rarityCache[set.set_rarity] = rows[0].id;
+                addedRarities++;
+              }
+              const rarity_id = rarityCache[set.set_rarity];
+
+              // Get set_id from cache
+              const set_id = setCache[set.set_code];
+              if (set_id) {
+                await conn.execute(`
+                  INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code)
+                  VALUES (?, ?, ?, ?)
+                  ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code)
+                `, [card.id, set_id, rarity_id, set.set_code]);
+              }
+            }
+          }
+
+          // Process card images
+          if (Array.isArray(card.card_images)) {
+            for (const img of card.card_images) {
+              await conn.execute(`
+                INSERT INTO card_images (card_id, image_url)
+                VALUES (?, ?)
+              `, [card.id, img.image_url]);
+              addedImages++;
+            }
+          }
+        }));
+
+        await conn.commit();
+      } catch (err) {
+        await conn.rollback();
+        throw err;
+      } finally {
+        conn.release();
+      }
+
+      // Release batch memory
+      batch.length = 0;
+    }
+
+    const endTime = Date.now();
+    const durationSeconds = ((endTime - startTime) / 1000).toFixed(2);
+
+    res.json({
+      total_cards: totalCards,
+      cards_added: addedCards,
+      images_added: addedImages,
+      rarities_added: addedRarities,
+      duration_seconds: parseFloat(durationSeconds)
+    });
+
+  } catch (err) {
+    console.error('Error importing cards:', err);
+    res.status(500).json({ error: 'Failed to import cards' });
+  }
+}
+
+module.exports = { importCards, importSets };

src/models/cardImageModel.js

@@ -0,0 +1,13 @@
+const db = require('../config/db');
+
+async function insertCardImage(image) {
+  const { card_id, image_url } = image;
+
+  const sql = `
+    INSERT INTO card_images (card_id, image_url)
+    VALUES (?, ?)
+  `;
+  await db.execute(sql, [card_id, image_url]);
+}
+
+module.exports = { insertCardImage };

src/models/cardModel.js

@@ -0,0 +1,34 @@
+const db = require('../config/db');
+
+async function upsertCard(card) {
+  const {
+    id,
+    name,
+    type,
+    frameType,
+    level,
+    race,
+    attribute,
+    linkval,
+    tcg_date,
+    ocg_date
+  } = card;
+
+  const sql = `
+    INSERT INTO cards (id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date)
+    VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+    ON DUPLICATE KEY UPDATE
+      name = VALUES(name),
+      card_type = VALUES(card_type),
+      frame_type = VALUES(frame_type),
+      level = VALUES(level),
+      race = VALUES(race),
+      attribute = VALUES(attribute),
+      link_val = VALUES(link_val),
+      tcg_date = VALUES(tcg_date),
+      ocg_date = VALUES(ocg_date)
+  `;
+  await db.execute(sql, [id, name, type, frameType, level, race, attribute, linkval, tcg_date, ocg_date]);
+}
+
+module.exports = { upsertCard };

src/models/cardSetRarityModel.js

@@ -0,0 +1,13 @@
+const db = require('../config/db');
+
+async function insertCardSetRarity(entry) {
+  const { card_id, set_id, rarity_id, card_set_code } = entry;
+  const sql = `
+    INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code)
+    VALUES (?, ?, ?, ?)
+    ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code)
+  `;
+  await db.execute(sql, [card_id, set_id, rarity_id, card_set_code]);
+}
+
+module.exports = { insertCardSetRarity };

src/models/rarityModel.js

@@ -0,0 +1,22 @@
+const db = require('../config/db');
+
+async function upsertRarity(rarity) {
+  const { rarity_name, rarity_code } = rarity;
+
+  const sql = `
+    INSERT INTO rarities (rarity_name, rarity_code)
+    VALUES (?, ?)
+    ON DUPLICATE KEY UPDATE
+      rarity_code = VALUES(rarity_code)
+  `;
+  const [result] = await db.execute(sql, [rarity_name, rarity_code]);
+  return result.insertId; // returns id for junction table
+}
+
+async function getRarityId(rarity_name) {
+  const sql = `SELECT id FROM rarities WHERE rarity_name = ? LIMIT 1`;
+  const [rows] = await db.execute(sql, [rarity_name]);
+  return rows.length ? rows[0].id : null;
+}
+
+module.exports = { upsertRarity, getRarityId };

src/routes/importRoutes.js

@@ -1,7 +1,8 @@
 const express = require('express');
 const router = express.Router();
-const { importSets } = require('../controllers/importController');
+const { importSets, importCards} = require('../controllers/importController');
 
 router.post('/sets', importSets);
+router.post('/cards', importCards);
 
 module.exports = router;

src/services/ygoproService.js

@@ -19,4 +19,15 @@ async function fetchAllSets() {
   }
 }
 
-module.exports = { fetchAllSets };
+async function fetchAllCards() {
+  try {
+    const response = await axios.get(`${API_BASE}/cardinfo.php?misc=yes`);
+    return response.data.data; // array of card objects
+  } catch (err) {
+    console.error('Error fetching cards from YGOPRODeck:', err.message);
+    throw err;
+  }
+}
+
+
+module.exports = { fetchAllCards, fetchAllSets };