Skip unchanged rows on import: diff cards, images, and set-rarities against DB state

src/controllers/importController.js

@@ -4,29 +4,31 @@ const db = require('../config/db');
 
 const BATCH_SIZE = 500;
 
+function cardFingerprint(c) {
+  return [c.name, c.type, c.frameType, c.level || null, c.race || null,
+          c.attribute || null, c.linkval || null, c.tcg_date || null, c.ocg_date || null].join('|');
+}
+
 async function importSetsInternal() {
   const sets = await fetchAllSets();
-  if (!sets.length) return { added: 0, total: 0 };
+  if (!sets.length) return { added: 0, total: sets.length };
 
-  const values = sets.map(() => '(?, ?, ?, ?)').join(', ');
-  const params = sets.flatMap(s => [s.set_name, s.set_code, s.num_of_cards, s.tcg_date]);
+  const [existing] = await db.execute('SELECT set_code FROM sets');
+  const existingCodes = new Set(existing.map(r => r.set_code));
 
-  await db.execute(`
-    INSERT INTO sets (set_name, set_code, num_of_cards, tcg_date)
-    VALUES ${values}
-    ON DUPLICATE KEY UPDATE
-      set_name = VALUES(set_name),
-      num_of_cards = VALUES(num_of_cards),
-      tcg_date = VALUES(tcg_date)
-  `, params);
+  const newSets = sets.filter(s => !existingCodes.has(s.set_code));
+  if (!newSets.length) return { added: 0, total: sets.length };
 
-  return { added: sets.length, total: sets.length };
+  const values = newSets.map(() => '(?, ?, ?, ?)').join(', ');
+  const params = newSets.flatMap(s => [s.set_name, s.set_code, s.num_of_cards, s.tcg_date]);
+  await db.execute(`INSERT INTO sets (set_name, set_code, num_of_cards, tcg_date) VALUES ${values}`, params);
+
+  return { added: newSets.length, total: sets.length };
 }
 
 async function importSets(req, res) {
   try {
-    const result = await importSetsInternal();
-    res.json(result);
+    res.json(await importSetsInternal());
   } catch (err) {
     console.error('Error importing sets:', err);
     res.status(500).json({ error: 'Failed to import sets' });
@@ -36,16 +38,31 @@ async function importSets(req, res) {
 async function importCardsInternal() {
   const startTime = Date.now();
   const cards = await fetchAllCards();
-  const totalCards = cards.length;
 
-  // Pre-load set and rarity caches
+  // Load full existing state for diffing
+  const [existingCards] = await db.execute(
+    'SELECT id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date FROM cards'
+  );
+  const cardFingerprintMap = new Map(existingCards.map(r => [
+    r.id,
+    [r.name, r.card_type, r.frame_type, r.level, r.race,
+     r.attribute, r.link_val, r.tcg_date, r.ocg_date].join('|')
+  ]));
+
+  const [existingImages] = await db.execute('SELECT image_id FROM card_images');
+  const existingImageIds = new Set(existingImages.map(r => r.image_id));
+
+  const [existingCSR] = await db.execute('SELECT card_id, set_id, rarity_id FROM card_sets_rarity');
+  const existingCSRKeys = new Set(existingCSR.map(r => `${r.card_id}-${r.set_id}-${r.rarity_id}`));
+
+  // Pre-load caches
   const [setsRows] = await db.execute('SELECT id, set_name FROM sets');
   const setCache = Object.fromEntries(setsRows.map(r => [r.set_name, r.id]));
 
   const [rarityRows] = await db.execute('SELECT id, rarity_name FROM rarities');
   const rarityCache = Object.fromEntries(rarityRows.map(r => [r.rarity_name, r.id]));
 
-  // Collect all unique rarities from the full card list and upsert them before the main loop
+  // Collect and upsert unseen rarities before the main loop
   const unseenRarities = new Map();
   for (const card of cards) {
     if (!Array.isArray(card.card_sets)) continue;
@@ -55,7 +72,6 @@ async function importCardsInternal() {
       }
     }
   }
-
   if (unseenRarities.size) {
     const rarityValues = Array.from(unseenRarities.keys()).map(() => '(?, ?)').join(', ');
     const rarityParams = Array.from(unseenRarities.entries()).flatMap(([name, code]) => [name, code]);
@@ -64,14 +80,13 @@ async function importCardsInternal() {
       VALUES ${rarityValues}
       ON DUPLICATE KEY UPDATE rarity_code = VALUES(rarity_code)
     `, rarityParams);
-
     const [newRarityRows] = await db.execute('SELECT id, rarity_name FROM rarities');
     for (const row of newRarityRows) rarityCache[row.rarity_name] = row.id;
   }
 
   let addedCards = 0, addedImages = 0, addedSetRarities = 0;
 
-  for (let i = 0; i < totalCards; i += BATCH_SIZE) {
+  for (let i = 0; i < cards.length; i += BATCH_SIZE) {
     const batch = cards.slice(i, i + BATCH_SIZE);
 
     const cardValues = [], cardParams = [];
@@ -79,17 +94,24 @@ async function importCardsInternal() {
     const csrValues = [], csrParams = [];
 
     for (const card of batch) {
+      const fp = cardFingerprint(card);
+      if (cardFingerprintMap.get(card.id) !== fp) {
         cardValues.push('(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
         cardParams.push(
           card.id, card.name, card.type, card.frameType,
           card.level || null, card.race || null, card.attribute || null,
           card.linkval || null, card.tcg_date || null, card.ocg_date || null
         );
+        addedCards++;
+      }
 
       if (Array.isArray(card.card_images)) {
         for (const img of card.card_images) {
+          if (!existingImageIds.has(img.id)) {
             imageValues.push('(?, ?, ?)');
             imageParams.push(img.id, card.id, img.image_url);
+            addedImages++;
+          }
         }
       }
 
@@ -98,17 +120,24 @@ async function importCardsInternal() {
           const set_id = setCache[set.set_name];
           const rarity_id = rarityCache[set.set_rarity];
           if (set_id && rarity_id) {
+            const key = `${card.id}-${set_id}-${rarity_id}`;
+            if (!existingCSRKeys.has(key)) {
               csrValues.push('(?, ?, ?, ?)');
               csrParams.push(card.id, set_id, rarity_id, set.set_code);
+              addedSetRarities++;
             }
           }
         }
       }
+    }
+
+    if (!cardValues.length && !imageValues.length && !csrValues.length) continue;
 
     const conn = await db.getConnection();
     try {
       await conn.beginTransaction();
 
+      if (cardValues.length) {
         await conn.execute(`
           INSERT INTO cards (id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date)
           VALUES ${cardValues.join(', ')}
@@ -117,27 +146,22 @@ async function importCardsInternal() {
             level = VALUES(level), race = VALUES(race), attribute = VALUES(attribute),
             link_val = VALUES(link_val), tcg_date = VALUES(tcg_date), ocg_date = VALUES(ocg_date)
         `, cardParams);
+      }
 
       if (imageValues.length) {
         await conn.execute(`
-          INSERT INTO card_images (image_id, card_id, image_url)
-          VALUES ${imageValues.join(', ')}
-          ON DUPLICATE KEY UPDATE image_url = VALUES(image_url)
+          INSERT INTO card_images (image_id, card_id, image_url) VALUES ${imageValues.join(', ')}
         `, imageParams);
       }
 
       if (csrValues.length) {
         await conn.execute(`
-          INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code)
-          VALUES ${csrValues.join(', ')}
+          INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code) VALUES ${csrValues.join(', ')}
           ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code)
         `, csrParams);
       }
 
       await conn.commit();
-      addedCards += batch.length;
-      addedImages += imageValues.length;
-      addedSetRarities += csrValues.length;
     } catch (err) {
       await conn.rollback();
       throw err;
@@ -147,7 +171,7 @@ async function importCardsInternal() {
   }
 
   return {
-    total_cards: totalCards,
+    total_cards: cards.length,
     cards_added: addedCards,
     images_added: addedImages,
     set_rarities_added: addedSetRarities,
@@ -157,8 +181,7 @@ async function importCardsInternal() {
 
 async function importCards(req, res) {
   try {
-    const result = await importCardsInternal();
-    res.json(result);
+    res.json(await importCardsInternal());
   } catch (err) {
     console.error('Error importing cards:', err);
     res.status(500).json({ error: 'Failed to import cards' });