256 lines
9.7 KiB
JavaScript
256 lines
9.7 KiB
JavaScript
const { getLocalDBVersion, setLocalDBVersion } = require('../models/dbVersionModel');
|
|
const { fetchAllCards, fetchAllSets, fetchDatabaseVersion } = require('../services/ygoproService');
|
|
const db = require('../config/db');
|
|
|
|
const BATCH_SIZE = 500;
|
|
|
|
function cardFingerprint(c) {
|
|
return [c.name, c.type, c.frameType, c.level || null, c.race || null,
|
|
c.attribute || null, c.linkval || null, c.tcg_date || null, c.ocg_date || null].join('|');
|
|
}
|
|
|
|
async function importSetsInternal() {
|
|
const sets = await fetchAllSets();
|
|
if (!sets.length) return { added: 0, total: sets.length };
|
|
|
|
const [existing] = await db.execute('SELECT set_code FROM sets');
|
|
const existingCodes = new Set(existing.map(r => r.set_code));
|
|
|
|
const newSets = sets.filter(s => !existingCodes.has(s.set_code));
|
|
if (!newSets.length) return { added: 0, total: sets.length };
|
|
|
|
const values = newSets.map(() => '(?, ?, ?, ?)').join(', ');
|
|
const params = newSets.flatMap(s => [s.set_name, s.set_code, s.num_of_cards, s.tcg_date]);
|
|
await db.execute(`INSERT INTO sets (set_name, set_code, num_of_cards, tcg_date) VALUES ${values}`, params);
|
|
|
|
return { added: newSets.length, total: sets.length };
|
|
}
|
|
|
|
async function importSets(req, res) {
|
|
try {
|
|
res.json(await importSetsInternal());
|
|
} catch (err) {
|
|
console.error('Error importing sets:', err);
|
|
res.status(500).json({ error: 'Failed to import sets' });
|
|
}
|
|
}
|
|
|
|
async function importCardsInternal(onProgress) {
|
|
const startTime = Date.now();
|
|
|
|
if (onProgress) onProgress({ message: 'Fetching cards from YGOPRODeck…', progress: 0.05 });
|
|
const cards = await fetchAllCards();
|
|
|
|
if (onProgress) onProgress({ message: 'Loading database state…', progress: 0.15 });
|
|
// Load full existing state for diffing
|
|
const [existingCards] = await db.execute(
|
|
'SELECT id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date FROM cards'
|
|
);
|
|
const cardFingerprintMap = new Map(existingCards.map(r => [
|
|
r.id,
|
|
[r.name, r.card_type, r.frame_type, r.level, r.race,
|
|
r.attribute, r.link_val, r.tcg_date, r.ocg_date].join('|')
|
|
]));
|
|
|
|
const [existingImages] = await db.execute('SELECT image_id FROM card_images');
|
|
const existingImageIds = new Set(existingImages.map(r => r.image_id));
|
|
|
|
const [existingCSR] = await db.execute('SELECT card_id, set_id, rarity_id FROM card_sets_rarity');
|
|
const existingCSRKeys = new Set(existingCSR.map(r => `${r.card_id}-${r.set_id}-${r.rarity_id}`));
|
|
|
|
// Pre-load caches
|
|
const [setsRows] = await db.execute('SELECT id, set_name FROM sets');
|
|
const setCache = Object.fromEntries(setsRows.map(r => [r.set_name, r.id]));
|
|
|
|
const [rarityRows] = await db.execute('SELECT id, rarity_name FROM rarities');
|
|
const rarityCache = Object.fromEntries(rarityRows.map(r => [r.rarity_name, r.id]));
|
|
|
|
// Collect and upsert unseen rarities before the main loop
|
|
const unseenRarities = new Map();
|
|
for (const card of cards) {
|
|
if (!Array.isArray(card.card_sets)) continue;
|
|
for (const set of card.card_sets) {
|
|
if (set.set_rarity && !rarityCache[set.set_rarity] && !unseenRarities.has(set.set_rarity)) {
|
|
unseenRarities.set(set.set_rarity, set.set_rarity_code);
|
|
}
|
|
}
|
|
}
|
|
if (unseenRarities.size) {
|
|
const rarityValues = Array.from(unseenRarities.keys()).map(() => '(?, ?)').join(', ');
|
|
const rarityParams = Array.from(unseenRarities.entries()).flatMap(([name, code]) => [name, code]);
|
|
await db.execute(`
|
|
INSERT INTO rarities (rarity_name, rarity_code)
|
|
VALUES ${rarityValues}
|
|
ON DUPLICATE KEY UPDATE rarity_code = VALUES(rarity_code)
|
|
`, rarityParams);
|
|
const [newRarityRows] = await db.execute('SELECT id, rarity_name FROM rarities');
|
|
for (const row of newRarityRows) rarityCache[row.rarity_name] = row.id;
|
|
}
|
|
|
|
let addedCards = 0, addedImages = 0, addedSetRarities = 0;
|
|
|
|
for (let i = 0; i < cards.length; i += BATCH_SIZE) {
|
|
const batch = cards.slice(i, i + BATCH_SIZE);
|
|
if (onProgress) onProgress({
|
|
message: `Processing ${Math.min(i + BATCH_SIZE, cards.length).toLocaleString()} / ${cards.length.toLocaleString()} cards…`,
|
|
progress: 0.2 + (i / cards.length) * 0.7,
|
|
});
|
|
|
|
const cardValues = [], cardParams = [];
|
|
const imageValues = [], imageParams = [];
|
|
const csrValues = [], csrParams = [];
|
|
const batchImageIds = new Set();
|
|
|
|
for (const card of batch) {
|
|
const fp = cardFingerprint(card);
|
|
if (cardFingerprintMap.get(card.id) !== fp) {
|
|
cardValues.push('(?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
|
|
cardParams.push(
|
|
card.id, card.name, card.type, card.frameType,
|
|
card.level || null, card.race || null, card.attribute || null,
|
|
card.linkval || null, card.tcg_date || null, card.ocg_date || null
|
|
);
|
|
addedCards++;
|
|
}
|
|
|
|
if (Array.isArray(card.card_images)) {
|
|
for (const img of card.card_images) {
|
|
if (!existingImageIds.has(img.id) && !batchImageIds.has(img.id)) {
|
|
batchImageIds.add(img.id);
|
|
imageValues.push('(?, ?, ?)');
|
|
imageParams.push(img.id, card.id, img.image_url);
|
|
addedImages++;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (Array.isArray(card.card_sets)) {
|
|
for (const set of card.card_sets) {
|
|
const set_id = setCache[set.set_name];
|
|
const rarity_id = rarityCache[set.set_rarity];
|
|
if (set_id && rarity_id) {
|
|
const key = `${card.id}-${set_id}-${rarity_id}`;
|
|
if (!existingCSRKeys.has(key)) {
|
|
csrValues.push('(?, ?, ?, ?)');
|
|
csrParams.push(card.id, set_id, rarity_id, set.set_code);
|
|
addedSetRarities++;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!cardValues.length && !imageValues.length && !csrValues.length) continue;
|
|
|
|
const conn = await db.getConnection();
|
|
try {
|
|
await conn.beginTransaction();
|
|
|
|
if (cardValues.length) {
|
|
await conn.execute(`
|
|
INSERT INTO cards (id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date)
|
|
VALUES ${cardValues.join(', ')}
|
|
ON DUPLICATE KEY UPDATE
|
|
name = VALUES(name), card_type = VALUES(card_type), frame_type = VALUES(frame_type),
|
|
level = VALUES(level), race = VALUES(race), attribute = VALUES(attribute),
|
|
link_val = VALUES(link_val), tcg_date = VALUES(tcg_date), ocg_date = VALUES(ocg_date)
|
|
`, cardParams);
|
|
}
|
|
|
|
if (imageValues.length) {
|
|
await conn.execute(`
|
|
INSERT INTO card_images (image_id, card_id, image_url) VALUES ${imageValues.join(', ')}
|
|
ON DUPLICATE KEY UPDATE image_url = VALUES(image_url)
|
|
`, imageParams);
|
|
}
|
|
|
|
if (csrValues.length) {
|
|
await conn.execute(`
|
|
INSERT INTO card_sets_rarity (card_id, set_id, rarity_id, card_set_code) VALUES ${csrValues.join(', ')}
|
|
ON DUPLICATE KEY UPDATE card_set_code = VALUES(card_set_code)
|
|
`, csrParams);
|
|
}
|
|
|
|
await conn.commit();
|
|
} catch (err) {
|
|
await conn.rollback();
|
|
throw err;
|
|
} finally {
|
|
conn.release();
|
|
}
|
|
}
|
|
|
|
if (onProgress) onProgress({ message: 'Cleaning up stale cards…', progress: 0.92 });
|
|
// Remove cards that no longer exist in the API
|
|
const apiCardIds = new Set(cards.map(c => c.id));
|
|
const staleIds = existingCards.map(r => r.id).filter(id => !apiCardIds.has(id));
|
|
if (staleIds.length) {
|
|
for (let i = 0; i < staleIds.length; i += 500) {
|
|
const chunk = staleIds.slice(i, i + 500);
|
|
const placeholders = chunk.map(() => '?').join(',');
|
|
await db.execute(`DELETE FROM card_images WHERE card_id IN (${placeholders})`, chunk);
|
|
await db.execute(`DELETE FROM card_sets_rarity WHERE card_id IN (${placeholders})`, chunk);
|
|
await db.execute(`DELETE FROM cards WHERE id IN (${placeholders})`, chunk);
|
|
}
|
|
}
|
|
|
|
return {
|
|
total_cards: cards.length,
|
|
cards_added: addedCards,
|
|
cards_removed: staleIds.length,
|
|
images_added: addedImages,
|
|
set_rarities_added: addedSetRarities,
|
|
duration_seconds: parseFloat(((Date.now() - startTime) / 1000).toFixed(2)),
|
|
};
|
|
}
|
|
|
|
async function importCards(req, res) {
|
|
try {
|
|
res.json(await importCardsInternal());
|
|
} catch (err) {
|
|
console.error('Error importing cards:', err);
|
|
res.status(500).json({ error: 'Failed to import cards' });
|
|
}
|
|
}
|
|
|
|
async function importFullDatabase(req, res) {
|
|
res.setHeader('Content-Type', 'text/event-stream');
|
|
res.setHeader('Cache-Control', 'no-cache');
|
|
res.setHeader('Connection', 'keep-alive');
|
|
res.flushHeaders();
|
|
|
|
const send = (data) => res.write(`data: ${JSON.stringify(data)}\n\n`);
|
|
const startTime = Date.now();
|
|
|
|
try {
|
|
send({ message: 'Checking version…', progress: 0 });
|
|
const [remoteVersion, localVersion] = await Promise.all([fetchDatabaseVersion(), getLocalDBVersion()]);
|
|
|
|
if (localVersion && localVersion.database_version === remoteVersion.database_version) {
|
|
send({ message: 'Already up to date.', progress: 1, done: true, version: remoteVersion.database_version });
|
|
return res.end();
|
|
}
|
|
|
|
send({ message: 'Importing sets…', progress: 0.02 });
|
|
const setsStats = await importSetsInternal();
|
|
|
|
const cardsStats = await importCardsInternal(send);
|
|
await setLocalDBVersion(remoteVersion.database_version, remoteVersion.last_update);
|
|
|
|
send({
|
|
message: 'Import complete!',
|
|
progress: 1,
|
|
done: true,
|
|
version: remoteVersion.database_version,
|
|
result: { ...cardsStats, duration_seconds: parseFloat(((Date.now() - startTime) / 1000).toFixed(2)) },
|
|
});
|
|
res.end();
|
|
} catch (err) {
|
|
console.error('Error in full database import:', err);
|
|
send({ message: `Error: ${err.message}`, progress: 0, done: true, error: true });
|
|
res.end();
|
|
}
|
|
}
|
|
|
|
module.exports = { importCards, importCardsInternal, importSets, importSetsInternal, importFullDatabase };
|