diff --git a/src/controllers/importController.js b/src/controllers/importController.js index 7474715..562b036 100644 --- a/src/controllers/importController.js +++ b/src/controllers/importController.js @@ -35,10 +35,13 @@ async function importSets(req, res) { } } -async function importCardsInternal() { +async function importCardsInternal(onProgress) { const startTime = Date.now(); + + if (onProgress) onProgress({ message: 'Fetching cards from YGOPRODeck…', progress: 0.05 }); const cards = await fetchAllCards(); + if (onProgress) onProgress({ message: 'Loading database state…', progress: 0.15 }); // Load full existing state for diffing const [existingCards] = await db.execute( 'SELECT id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date FROM cards' @@ -88,6 +91,10 @@ async function importCardsInternal() { for (let i = 0; i < cards.length; i += BATCH_SIZE) { const batch = cards.slice(i, i + BATCH_SIZE); + if (onProgress) onProgress({ + message: `Processing ${Math.min(i + BATCH_SIZE, cards.length).toLocaleString()} / ${cards.length.toLocaleString()} cards…`, + progress: 0.2 + (i / cards.length) * 0.7, + }); const cardValues = [], cardParams = []; const imageValues = [], imageParams = []; @@ -173,6 +180,7 @@ async function importCardsInternal() { } } + if (onProgress) onProgress({ message: 'Cleaning up stale cards…', progress: 0.92 }); // Remove cards that no longer exist in the API const apiCardIds = new Set(cards.map(c => c.id)); const staleIds = existingCards.map(r => r.id).filter(id => !apiCardIds.has(id)); @@ -206,29 +214,41 @@ async function importCards(req, res) { } async function importFullDatabase(req, res) { + res.setHeader('Content-Type', 'text/event-stream'); + res.setHeader('Cache-Control', 'no-cache'); + res.setHeader('Connection', 'keep-alive'); + res.flushHeaders(); + + const send = (data) => res.write(`data: ${JSON.stringify(data)}\n\n`); const startTime = Date.now(); + try { - const remoteVersion = await fetchDatabaseVersion(); - const localVersion = await getLocalDBVersion(); + send({ message: 'Checking version…', progress: 0 }); + const [remoteVersion, localVersion] = await Promise.all([fetchDatabaseVersion(), getLocalDBVersion()]); if (localVersion && localVersion.database_version === remoteVersion.database_version) { - return res.json({ message: 'Database is already up to date.', version: remoteVersion.database_version }); + send({ message: 'Already up to date.', progress: 1, done: true, version: remoteVersion.database_version }); + return res.end(); } + send({ message: 'Importing sets…', progress: 0.02 }); const setsStats = await importSetsInternal(); - const cardsStats = await importCardsInternal(); + + const cardsStats = await importCardsInternal(send); await setLocalDBVersion(remoteVersion.database_version, remoteVersion.last_update); - res.json({ - message: 'Database import completed successfully.', + send({ + message: 'Import complete!', + progress: 1, + done: true, version: remoteVersion.database_version, - sets: setsStats, - cards: cardsStats, - duration_seconds: parseFloat(((Date.now() - startTime) / 1000).toFixed(2)), + result: { ...cardsStats, duration_seconds: parseFloat(((Date.now() - startTime) / 1000).toFixed(2)) }, }); + res.end(); } catch (err) { console.error('Error in full database import:', err); - res.status(500).json({ error: 'Failed to import database' }); + send({ message: `Error: ${err.message}`, progress: 0, done: true, error: true }); + res.end(); } }