Stream import progress via SSE

src/controllers/importController.js

@@ -35,10 +35,13 @@ async function importSets(req, res) {
   }
 }
 
-async function importCardsInternal() {
+async function importCardsInternal(onProgress) {
   const startTime = Date.now();
+
+  if (onProgress) onProgress({ message: 'Fetching cards from YGOPRODeck…', progress: 0.05 });
   const cards = await fetchAllCards();
 
+  if (onProgress) onProgress({ message: 'Loading database state…', progress: 0.15 });
   // Load full existing state for diffing
   const [existingCards] = await db.execute(
     'SELECT id, name, card_type, frame_type, level, race, attribute, link_val, tcg_date, ocg_date FROM cards'
@@ -88,6 +91,10 @@ async function importCardsInternal() {
 
   for (let i = 0; i < cards.length; i += BATCH_SIZE) {
     const batch = cards.slice(i, i + BATCH_SIZE);
+    if (onProgress) onProgress({
+      message: `Processing ${Math.min(i + BATCH_SIZE, cards.length).toLocaleString()} / ${cards.length.toLocaleString()} cards…`,
+      progress: 0.2 + (i / cards.length) * 0.7,
+    });
 
     const cardValues = [], cardParams = [];
     const imageValues = [], imageParams = [];
@@ -173,6 +180,7 @@ async function importCardsInternal() {
     }
   }
 
+  if (onProgress) onProgress({ message: 'Cleaning up stale cards…', progress: 0.92 });
   // Remove cards that no longer exist in the API
   const apiCardIds = new Set(cards.map(c => c.id));
   const staleIds = existingCards.map(r => r.id).filter(id => !apiCardIds.has(id));
@@ -206,29 +214,41 @@ async function importCards(req, res) {
 }
 
 async function importFullDatabase(req, res) {
+  res.setHeader('Content-Type', 'text/event-stream');
+  res.setHeader('Cache-Control', 'no-cache');
+  res.setHeader('Connection', 'keep-alive');
+  res.flushHeaders();
+
+  const send = (data) => res.write(`data: ${JSON.stringify(data)}\n\n`);
   const startTime = Date.now();
+
   try {
-    const remoteVersion = await fetchDatabaseVersion();
-    const localVersion = await getLocalDBVersion();
+    send({ message: 'Checking version…', progress: 0 });
+    const [remoteVersion, localVersion] = await Promise.all([fetchDatabaseVersion(), getLocalDBVersion()]);
 
     if (localVersion && localVersion.database_version === remoteVersion.database_version) {
-      return res.json({ message: 'Database is already up to date.', version: remoteVersion.database_version });
+      send({ message: 'Already up to date.', progress: 1, done: true, version: remoteVersion.database_version });
+      return res.end();
     }
 
+    send({ message: 'Importing sets…', progress: 0.02 });
     const setsStats = await importSetsInternal();
-    const cardsStats = await importCardsInternal();
+
+    const cardsStats = await importCardsInternal(send);
     await setLocalDBVersion(remoteVersion.database_version, remoteVersion.last_update);
 
-    res.json({
-      message: 'Database import completed successfully.',
+    send({
+      message: 'Import complete!',
+      progress: 1,
+      done: true,
       version: remoteVersion.database_version,
-      sets: setsStats,
-      cards: cardsStats,
-      duration_seconds: parseFloat(((Date.now() - startTime) / 1000).toFixed(2)),
+      result: { ...cardsStats, duration_seconds: parseFloat(((Date.now() - startTime) / 1000).toFixed(2)) },
     });
+    res.end();
   } catch (err) {
     console.error('Error in full database import:', err);
-    res.status(500).json({ error: 'Failed to import database' });
+    send({ message: `Error: ${err.message}`, progress: 0, done: true, error: true });
+    res.end();
   }
 }