From 5e1c2558e7866dcf77f11aaa082d2f2bd3ce1a26 Mon Sep 17 00:00:00 2001 From: Syco21 Date: Fri, 15 May 2026 21:59:26 +0200 Subject: [PATCH] Skip unchanged rows on import: diff cards, images, and set-rarities against DB state --- src/controllers/importController.js | 121 +++++++++++++++++----------- 1 file changed, 72 insertions(+), 49 deletions(-) diff --git a/src/controllers/importController.js b/src/controllers/importController.js index 2d76d68..4c05cb7 100644 --- a/src/controllers/importController.js +++ b/src/controllers/importController.js @@ -4,29 +4,31 @@ const db = require('../config/db'); const BATCH_SIZE = 500; +function cardFingerprint(c) { + return [c.name, c.type, c.frameType, c.level || null, c.race || null, + c.attribute || null, c.linkval || null, c.tcg_date || null, c.ocg_date || null].join('|'); +} + async function importSetsInternal() { const sets = await fetchAllSets(); - if (!sets.length) return { added: 0, total: 0 }; + if (!sets.length) return { added: 0, total: sets.length }; - const values = sets.map(() => '(?, ?, ?, ?)').join(', '); - const params = sets.flatMap(s => [s.set_name, s.set_code, s.num_of_cards, s.tcg_date]); + const [existing] = await db.execute('SELECT set_code FROM sets'); + const existingCodes = new Set(existing.map(r => r.set_code)); - await db.execute(` - INSERT INTO sets (set_name, set_code, num_of_cards, tcg_date) - VALUES ${values} - ON DUPLICATE KEY UPDATE - set_name = VALUES(set_name), - num_of_cards = VALUES(num_of_cards), - tcg_date = VALUES(tcg_date) - `, params); + const newSets = sets.filter(s => !existingCodes.has(s.set_code)); + if (!newSets.length) return { added: 0, total: sets.length }; - return { added: sets.length, total: sets.length }; + const values = newSets.map(() => '(?, ?, ?, ?)').join(', '); + const params = newSets.flatMap(s => [s.set_name, s.set_code, s.num_of_cards, s.tcg_date]); + await db.execute(`INSERT INTO sets (set_name, set_code, num_of_cards, tcg_date) VALUES ${values}`, params); + + return { added: newSets.length, total: sets.length }; } async function importSets(req, res) { try { - const result = await importSetsInternal(); - res.json(result); + res.json(await importSetsInternal()); } catch (err) { console.error('Error importing sets:', err); res.status(500).json({ error: 'Failed to import sets' }); @@ -36,16 +38,31 @@ async function importSets(req, res) { async function importCardsInternal() { const startTime = Date.now(); const cards = await fetchAllCards(); - const totalCards = cards.length; - // Pre-load set and rarity caches + // Load full existing state for diffing + const [existingCards] = await db.execute( + 'SELECT id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date FROM cards' + ); + const cardFingerprintMap = new Map(existingCards.map(r => [ + r.id, + [r.name, r.card_type, r.frame_type, r.level, r.race, + r.attribute, r.link_val, r.tcg_date, r.ocg_date].join('|') + ])); + + const [existingImages] = await db.execute('SELECT image_id FROM card_images'); + const existingImageIds = new Set(existingImages.map(r => r.image_id)); + + const [existingCSR] = await db.execute('SELECT card_id, set_id, rarity_id FROM card_sets_rarity'); + const existingCSRKeys = new Set(existingCSR.map(r => `${r.card_id}-${r.set_id}-${r.rarity_id}`)); + + // Pre-load caches const [setsRows] = await db.execute('SELECT id, set_name FROM sets'); const setCache = Object.fromEntries(setsRows.map(r => [r.set_name, r.id])); const [rarityRows] = await db.execute('SELECT id, rarity_name FROM rarities'); const rarityCache = Object.fromEntries(rarityRows.map(r => [r.rarity_name, r.id])); - // Collect all unique rarities from the full card list and upsert them before the main loop + // Collect and upsert unseen rarities before the main loop const unseenRarities = new Map(); for (const card of cards) { if (!Array.isArray(card.card_sets)) continue; @@ -55,7 +72,6 @@ async function importCardsInternal() { } } } - if (unseenRarities.size) { const rarityValues = Array.from(unseenRarities.keys()).map(() => '(?, ?)').join(', '); const rarityParams = Array.from(unseenRarities.entries()).flatMap(([name, code]) => [name, code]); @@ -64,14 +80,13 @@ async function importCardsInternal() { VALUES ${rarityValues} ON DUPLICATE KEY UPDATE rarity_code = VALUES(rarity_code) `, rarityParams); - const [newRarityRows] = await db.execute('SELECT id, rarity_name FROM rarities'); for (const row of newRarityRows) rarityCache[row.rarity_name] = row.id; } let addedCards = 0, addedImages = 0, addedSetRarities = 0; - for (let i = 0; i < totalCards; i += BATCH_SIZE) { + for (let i = 0; i < cards.length; i += BATCH_SIZE) { const batch = cards.slice(i, i + BATCH_SIZE); const cardValues = [], cardParams = []; @@ -79,17 +94,24 @@ async function importCardsInternal() { const csrValues = [], csrParams = []; for (const card of batch) { - cardValues.push('(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'); - cardParams.push( - card.id, card.name, card.type, card.frameType, - card.level || null, card.race || null, card.attribute || null, - card.linkval || null, card.tcg_date || null, card.ocg_date || null - ); + const fp = cardFingerprint(card); + if (cardFingerprintMap.get(card.id) !== fp) { + cardValues.push('(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)'); + cardParams.push( + card.id, card.name, card.type, card.frameType, + card.level || null, card.race || null, card.attribute || null, + card.linkval || null, card.tcg_date || null, card.ocg_date || null + ); + addedCards++; + } if (Array.isArray(card.card_images)) { for (const img of card.card_images) { - imageValues.push('(?, ?, ?)'); - imageParams.push(img.id, card.id, img.image_url); + if (!existingImageIds.has(img.id)) { + imageValues.push('(?, ?, ?)'); + imageParams.push(img.id, card.id, img.image_url); + addedImages++; + } } } @@ -98,46 +120,48 @@ async function importCardsInternal() { const set_id = setCache[set.set_name]; const rarity_id = rarityCache[set.set_rarity]; if (set_id && rarity_id) { - csrValues.push('(?, ?, ?, ?)'); - csrParams.push(card.id, set_id, rarity_id, set.set_code); + const key = `${card.id}-${set_id}-${rarity_id}`; + if (!existingCSRKeys.has(key)) { + csrValues.push('(?, ?, ?, ?)'); + csrParams.push(card.id, set_id, rarity_id, set.set_code); + addedSetRarities++; + } } } } } + if (!cardValues.length && !imageValues.length && !csrValues.length) continue; + const conn = await db.getConnection(); try { await conn.beginTransaction(); - await conn.execute(` - INSERT INTO cards (id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date) - VALUES ${cardValues.join(', ')} - ON DUPLICATE KEY UPDATE - name = VALUES(name), card_type = VALUES(card_type), frame_type = VALUES(frame_type), - level = VALUES(level), race = VALUES(race), attribute = VALUES(attribute), - link_val = VALUES(link_val), tcg_date = VALUES(tcg_date), ocg_date = VALUES(ocg_date) - `, cardParams); + if (cardValues.length) { + await conn.execute(` + INSERT INTO cards (id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date) + VALUES ${cardValues.join(', ')} + ON DUPLICATE KEY UPDATE + name = VALUES(name), card_type = VALUES(card_type), frame_type = VALUES(frame_type), + level = VALUES(level), race = VALUES(race), attribute = VALUES(attribute), + link_val = VALUES(link_val), tcg_date = VALUES(tcg_date), ocg_date = VALUES(ocg_date) + `, cardParams); + } if (imageValues.length) { await conn.execute(` - INSERT INTO card_images (image_id, card_id, image_url) - VALUES ${imageValues.join(', ')} - ON DUPLICATE KEY UPDATE image_url = VALUES(image_url) + INSERT INTO card_images (image_id, card_id, image_url) VALUES ${imageValues.join(', ')} `, imageParams); } if (csrValues.length) { await conn.execute(` - INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code) - VALUES ${csrValues.join(', ')} + INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code) VALUES ${csrValues.join(', ')} ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code) `, csrParams); } await conn.commit(); - addedCards += batch.length; - addedImages += imageValues.length; - addedSetRarities += csrValues.length; } catch (err) { await conn.rollback(); throw err; @@ -147,7 +171,7 @@ async function importCardsInternal() { } return { - total_cards: totalCards, + total_cards: cards.length, cards_added: addedCards, images_added: addedImages, set_rarities_added: addedSetRarities, @@ -157,8 +181,7 @@ async function importCardsInternal() { async function importCards(req, res) { try { - const result = await importCardsInternal(); - res.json(result); + res.json(await importCardsInternal()); } catch (err) { console.error('Error importing cards:', err); res.status(500).json({ error: 'Failed to import cards' });