Fix: Add batch processing to prevent timeout on large song libraries

app/admin/page.tsx

@@ -134,19 +134,56 @@ export default function AdminPage() {
         setCategorizationResults(null);
 
         try {
+            let offset = 0;
+            let hasMore = true;
+            let allResults: any[] = [];
+            let totalUncategorized = 0;
+            let totalProcessed = 0;
+
+            // Process in batches
+            while (hasMore) {
                 const res = await fetch('/api/categorize', {
                     method: 'POST',
+                    headers: { 'Content-Type': 'application/json' },
+                    body: JSON.stringify({ offset })
                 });
 
-            if (res.ok) {
+                if (!res.ok) {
-                const data = await res.json();
-                setCategorizationResults(data);
-                fetchSongs(); // Refresh song list
-                fetchGenres(); // Refresh genre counts
-            } else {
                     const error = await res.json();
                     alert(`Categorization failed: ${error.error || 'Unknown error'}`);
+                    break;
                 }
+
+                const data = await res.json();
+                totalUncategorized = data.totalUncategorized;
+                totalProcessed = data.processed;
+                hasMore = data.hasMore;
+                offset = data.nextOffset || 0;
+
+                // Accumulate results
+                allResults = [...allResults, ...data.results];
+
+                // Update UI with progress
+                setCategorizationResults({
+                    message: `Processing: ${totalProcessed} / ${totalUncategorized} songs...`,
+                    totalProcessed: totalUncategorized,
+                    totalCategorized: allResults.length,
+                    results: allResults,
+                    inProgress: hasMore
+                });
+            }
+
+            // Final update
+            setCategorizationResults({
+                message: `Completed! Processed ${totalUncategorized} songs, categorized ${allResults.length}`,
+                totalProcessed: totalUncategorized,
+                totalCategorized: allResults.length,
+                results: allResults,
+                inProgress: false
+            });
+
+            fetchSongs(); // Refresh song list
+            fetchGenres(); // Refresh genre counts
         } catch (error) {
             alert('Failed to categorize songs');
             console.error(error);

app/api/categorize/route.ts

@@ -6,6 +6,7 @@ const prisma = new PrismaClient();
 
 const OPENROUTER_API_KEY = process.env.OPENROUTER_API_KEY;
 const OPENROUTER_MODEL = 'anthropic/claude-3.5-haiku';
+const BATCH_SIZE = 20; // Process 20 songs at a time to avoid timeouts
 
 interface CategorizeResult {
     songId: number;
@@ -14,7 +15,7 @@ interface CategorizeResult {
     assignedGenres: string[];
 }
 
-export async function POST() {
+export async function POST(request: Request) {
     try {
         if (!OPENROUTER_API_KEY) {
             return Response.json(
@@ -23,7 +24,30 @@ export async function POST() {
             );
         }
 
+        // Get offset from request body (for batch processing)
+        const body = await request.json().catch(() => ({}));
+        const offset = body.offset || 0;
+
         // Fetch all songs without genres
+        const totalUncategorized = await prisma.song.count({
+            where: {
+                genres: {
+                    none: {}
+                }
+            }
+        });
+
+        if (totalUncategorized === 0) {
+            return Response.json({
+                message: 'No uncategorized songs found',
+                results: [],
+                hasMore: false,
+                totalUncategorized: 0,
+                processed: 0
+            });
+        }
+
+        // Fetch batch of songs
         const uncategorizedSongs = await prisma.song.findMany({
             where: {
                 genres: {
@@ -32,16 +56,11 @@ export async function POST() {
             },
             include: {
                 genres: true
-            }
+            },
+            take: BATCH_SIZE,
+            skip: offset
         });
 
-        if (uncategorizedSongs.length === 0) {
-            return Response.json({
-                message: 'No uncategorized songs found',
-                results: []
-            });
-        }
-
         // Fetch all available genres
         const allGenres = await prisma.genre.findMany({
             orderBy: { name: 'asc' }
@@ -56,7 +75,7 @@ export async function POST() {
 
         const results: CategorizeResult[] = [];
 
-        // Process each song
+        // Process each song in this batch
         for (const song of uncategorizedSongs) {
             try {
                 const genreNames = allGenres.map(g => g.name);
@@ -147,10 +166,15 @@ Your response:`;
             }
         }
 
+        const newOffset = offset + BATCH_SIZE;
+        const hasMore = newOffset < totalUncategorized;
+
         return Response.json({
-            message: `Processed ${uncategorizedSongs.length} songs, categorized ${results.length}`,
+            message: `Processed ${uncategorizedSongs.length} songs in this batch, categorized ${results.length}`,
-            totalProcessed: uncategorizedSongs.length,
+            totalUncategorized,
-            totalCategorized: results.length,
+            processed: Math.min(newOffset, totalUncategorized),
+            hasMore,
+            nextOffset: hasMore ? newOffset : null,
             results
         });