compare versions
Compare veresions and update sets -> Cards
This commit is contained in:
+126
-134
@@ -1,172 +1,164 @@
|
||||
const {
|
||||
upsertCard
|
||||
} = require('../models/cardModel');
|
||||
const {
|
||||
upsertRarity,
|
||||
getRarityId
|
||||
} = require('../models/rarityModel');
|
||||
const {upsertCard} = require('../models/cardModel');
|
||||
const {upsertRarity,getRarityId} = require('../models/rarityModel');
|
||||
const { upsertSet } = require('../models/setModel');
|
||||
const { insertCardSetRarity } = require('../models/cardSetRarityModel');
|
||||
const { insertCardImage } = require('../models/cardImageModel');
|
||||
const { fetchAllCards } = require('../services/ygoproService');
|
||||
const { fetchAllCards, fetchAllSets } = require('../services/ygoproService');
|
||||
const db = require('../config/db');
|
||||
|
||||
const BATCH_SIZE = 50;
|
||||
|
||||
async function importSetsInternal() {
|
||||
const sets = await fetchAllSets();
|
||||
let added = 0;
|
||||
|
||||
for (let i = 0; i < sets.length; i += BATCH_SIZE) {
|
||||
const batch = sets.slice(i, i + BATCH_SIZE);
|
||||
await Promise.all(batch.map(async (set) => {
|
||||
await upsertSet(set);
|
||||
added++;
|
||||
}));
|
||||
}
|
||||
|
||||
return {
|
||||
added,
|
||||
total: sets.length
|
||||
};
|
||||
}
|
||||
|
||||
// Express handler for /sets endpoint
|
||||
async function importSets(req, res) {
|
||||
try {
|
||||
const sets = await fetchAllSets();
|
||||
let added = 0;
|
||||
|
||||
// Parallelized batch insert
|
||||
for (let i = 0; i < sets.length; i += BATCH_SIZE) {
|
||||
const batch = sets.slice(i, i + BATCH_SIZE);
|
||||
await Promise.all(batch.map(async (set) => {
|
||||
await upsertSet(set);
|
||||
added++;
|
||||
}));
|
||||
}
|
||||
|
||||
res.json({
|
||||
added,
|
||||
total: sets.length
|
||||
});
|
||||
const result = await importSetsInternal();
|
||||
res.json(result);
|
||||
} catch (err) {
|
||||
console.error('Error importing sets:', err);
|
||||
res.status(500).json({ error: 'Failed to import sets' });
|
||||
}
|
||||
}
|
||||
|
||||
async function importCards(req, res) {
|
||||
async function importCardsInternal() {
|
||||
const startTime = Date.now();
|
||||
const cards = await fetchAllCards();
|
||||
const totalCards = cards.length;
|
||||
|
||||
try {
|
||||
const cards = await fetchAllCards();
|
||||
const totalCards = cards.length;
|
||||
let addedCards = 0, addedImages = 0, addedRarities = 0;
|
||||
|
||||
let addedCards = 0, addedImages = 0, addedRarities = 0;
|
||||
const rarityCache = {};
|
||||
const setCache = {};
|
||||
|
||||
// Cache sets and rarities to reduce repeated DB lookups
|
||||
const rarityCache = {};
|
||||
const setCache = {};
|
||||
const [setsRows] = await db.execute('SELECT id, set_name FROM sets');
|
||||
for (const row of setsRows) {
|
||||
setCache[row.set_name] = row.id;
|
||||
}
|
||||
|
||||
// Preload sets into cache
|
||||
const [setsRows] = await db.execute('SELECT id, set_code FROM sets');
|
||||
for (const row of setsRows) {
|
||||
setCache[row.set_code] = row.id;
|
||||
}
|
||||
const [rarityRows] = await db.execute('SELECT id, rarity_name FROM rarities');
|
||||
for (const row of rarityRows) {
|
||||
rarityCache[row.rarity_name] = row.id;
|
||||
}
|
||||
|
||||
// Preload rarities into cache
|
||||
const [rarityRows] = await db.execute('SELECT id, rarity_name FROM rarities');
|
||||
for (const row of rarityRows) {
|
||||
rarityCache[row.rarity_name] = row.id;
|
||||
}
|
||||
for (let i = 0; i < totalCards; i += BATCH_SIZE) {
|
||||
const batch = cards.slice(i, i + BATCH_SIZE);
|
||||
const conn = await db.getConnection();
|
||||
try {
|
||||
await conn.beginTransaction();
|
||||
|
||||
// Process cards in batches
|
||||
for (let i = 0; i < totalCards; i += BATCH_SIZE) {
|
||||
const batch = cards.slice(i, i + BATCH_SIZE);
|
||||
await Promise.all(batch.map(async (card) => {
|
||||
await conn.execute(`
|
||||
INSERT INTO cards
|
||||
(id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
name = VALUES(name),
|
||||
card_type = VALUES(card_type),
|
||||
frame_type = VALUES(frame_type),
|
||||
level = VALUES(level),
|
||||
race = VALUES(race),
|
||||
attribute = VALUES(attribute),
|
||||
link_val = VALUES(link_val),
|
||||
tcg_date = VALUES(tcg_date),
|
||||
ocg_date = VALUES(ocg_date)
|
||||
`, [
|
||||
card.id,
|
||||
card.name,
|
||||
card.type,
|
||||
card.frameType,
|
||||
card.level || null,
|
||||
card.race || null,
|
||||
card.attribute || null,
|
||||
card.linkval || null,
|
||||
card.tcg_date || null,
|
||||
card.ocg_date || null
|
||||
]);
|
||||
addedCards++;
|
||||
|
||||
// Wrap batch in a transaction
|
||||
const conn = await db.getConnection();
|
||||
try {
|
||||
await conn.beginTransaction();
|
||||
|
||||
await Promise.all(batch.map(async (card) => {
|
||||
// Upsert card
|
||||
await conn.execute(`
|
||||
INSERT INTO cards
|
||||
(id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE
|
||||
name = VALUES(name),
|
||||
card_type = VALUES(card_type),
|
||||
frame_type = VALUES(frame_type),
|
||||
level = VALUES(level),
|
||||
race = VALUES(race),
|
||||
attribute = VALUES(attribute),
|
||||
link_val = VALUES(link_val),
|
||||
tcg_date = VALUES(tcg_date),
|
||||
ocg_date = VALUES(ocg_date)
|
||||
`, [
|
||||
card.id,
|
||||
card.name,
|
||||
card.type,
|
||||
card.frameType,
|
||||
card.level || null,
|
||||
card.race || null,
|
||||
card.attribute || null,
|
||||
card.linkval || null,
|
||||
card.tcg_date || null,
|
||||
card.ocg_date || null
|
||||
]);
|
||||
addedCards++;
|
||||
|
||||
// Process rarities and card_sets_rarity
|
||||
if (Array.isArray(card.card_sets)) {
|
||||
for (const set of card.card_sets) {
|
||||
// Upsert rarity if not cached
|
||||
if (!rarityCache[set.set_rarity]) {
|
||||
await conn.execute(`
|
||||
INSERT INTO rarities (rarity_name, rarity_code)
|
||||
VALUES (?, ?)
|
||||
ON DUPLICATE KEY UPDATE rarity_code = VALUES(rarity_code)
|
||||
`, [set.set_rarity, set.set_rarity_code]);
|
||||
const [rows] = await conn.execute('SELECT id FROM rarities WHERE rarity_name = ? LIMIT 1', [set.set_rarity]);
|
||||
rarityCache[set.set_rarity] = rows[0].id;
|
||||
addedRarities++;
|
||||
}
|
||||
const rarity_id = rarityCache[set.set_rarity];
|
||||
|
||||
// Get set_id from cache
|
||||
const set_id = setCache[set.set_code];
|
||||
if (set_id) {
|
||||
await conn.execute(`
|
||||
INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code)
|
||||
`, [card.id, set_id, rarity_id, set.set_code]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process card images
|
||||
if (Array.isArray(card.card_images)) {
|
||||
for (const img of card.card_images) {
|
||||
if (Array.isArray(card.card_sets)) {
|
||||
for (const set of card.card_sets) {
|
||||
if (!rarityCache[set.set_rarity]) {
|
||||
await conn.execute(`
|
||||
INSERT INTO card_images (card_id, image_url)
|
||||
INSERT INTO rarities (rarity_name, rarity_code)
|
||||
VALUES (?, ?)
|
||||
`, [card.id, img.image_url]);
|
||||
addedImages++;
|
||||
ON DUPLICATE KEY UPDATE rarity_code = VALUES(rarity_code)
|
||||
`, [set.set_rarity, set.set_rarity_code]);
|
||||
const [rows] = await conn.execute('SELECT id FROM rarities WHERE rarity_name = ? LIMIT 1', [set.set_rarity]);
|
||||
rarityCache[set.set_rarity] = rows[0].id;
|
||||
addedRarities++;
|
||||
}
|
||||
const rarity_id = rarityCache[set.set_rarity];
|
||||
const set_id = setCache[set.set_name];
|
||||
if (set_id) {
|
||||
await conn.execute(`
|
||||
INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code)
|
||||
VALUES (?, ?, ?, ?)
|
||||
ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code)
|
||||
`, [card.id, set_id, rarity_id, set.set_code]);
|
||||
}
|
||||
}
|
||||
}));
|
||||
}
|
||||
|
||||
await conn.commit();
|
||||
} catch (err) {
|
||||
await conn.rollback();
|
||||
throw err;
|
||||
} finally {
|
||||
conn.release();
|
||||
}
|
||||
if (Array.isArray(card.card_images)) {
|
||||
for (const img of card.card_images) {
|
||||
await conn.execute(`
|
||||
INSERT INTO card_images (card_id, image_url)
|
||||
VALUES (?, ?)
|
||||
`, [card.id, img.image_url]);
|
||||
addedImages++;
|
||||
}
|
||||
}
|
||||
}));
|
||||
|
||||
// Release batch memory
|
||||
batch.length = 0;
|
||||
await conn.commit();
|
||||
} catch (err) {
|
||||
await conn.rollback();
|
||||
throw err;
|
||||
} finally {
|
||||
conn.release();
|
||||
}
|
||||
|
||||
const endTime = Date.now();
|
||||
const durationSeconds = ((endTime - startTime) / 1000).toFixed(2);
|
||||
batch.length = 0;
|
||||
}
|
||||
|
||||
res.json({
|
||||
total_cards: totalCards,
|
||||
cards_added: addedCards,
|
||||
images_added: addedImages,
|
||||
rarities_added: addedRarities,
|
||||
duration_seconds: parseFloat(durationSeconds)
|
||||
});
|
||||
const durationSeconds = ((Date.now() - startTime) / 1000).toFixed(2);
|
||||
|
||||
return {
|
||||
total_cards: totalCards,
|
||||
cards_added: addedCards,
|
||||
images_added: addedImages,
|
||||
rarities_added: addedRarities,
|
||||
duration_seconds: parseFloat(durationSeconds)
|
||||
};
|
||||
}
|
||||
|
||||
// Express handler for /cards
|
||||
async function importCards(req, res) {
|
||||
try {
|
||||
const result = await importCardsInternal();
|
||||
res.json(result);
|
||||
} catch (err) {
|
||||
console.error('Error importing cards:', err);
|
||||
res.status(500).json({ error: 'Failed to import cards' });
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { importCards, importSets };
|
||||
|
||||
module.exports = { importCards, importCardsInternal, importSets, importSetsInternal };
|
||||
Reference in New Issue
Block a user