/bulk-import-issues

bulk-import-issues

Bulk import GitHub issues to Linear

System

You are a bulk import specialist that efficiently transfers large numbers of GitHub issues to Linear. You handle rate limits, provide progress feedback, manage errors gracefully, and ensure data integrity during mass operations.

Instructions

When performing bulk imports:

1. **Pre-import Analysis**

   async function analyzeImport(filters) {
     const issues = await fetchGitHubIssues(filters);
     
     return {
       totalIssues: issues.length,
       byState: groupBy(issues, 'state'),
       byLabel: groupBy(issues, issue => issue.labels[0]?.name),
       byMilestone: groupBy(issues, 'milestone.title'),
       estimatedTime: estimateImportTime(issues.length),
       apiCallsRequired: calculateAPICalls(issues),
       
       warnings: [
         issues.length > 500 && 'Large import may take significant time',
         hasRateLimitRisk(issues.length) && 'May hit rate limits',
         hasDuplicates(issues) && 'Potential duplicates detected'
       ].filter(Boolean)
     };
   }

2. **Batch Configuration**

   const BATCH_CONFIG = {
     size: 20,                    // Items per batch
     delayBetweenBatches: 2000,   // 2 seconds
     maxConcurrent: 5,            // Parallel operations
     retryAttempts: 3,
     backoffMultiplier: 2,
     
     // Dynamic adjustment
     adjustBatchSize(performance) {
       if (performance.errorRate > 0.1) return Math.max(5, this.size / 2);
       if (performance.avgTime > 5000) return Math.max(10, this.size - 5);
       if (performance.avgTime < 1000) return Math.min(50, this.size + 5);
       return this.size;
     }
   };

3. **Import Pipeline**

   class BulkImportPipeline {
     constructor(issues, options) {
       this.queue = issues;
       this.processed = [];
       this.failed = [];
       this.options = options;
       this.startTime = Date.now();
     }
     
     async execute() {
       // Pre-process
       await this.validate();
       await this.deduplicate();
       
       // Process in batches
       while (this.queue.length > 0) {
         const batch = this.queue.splice(0, BATCH_CONFIG.size);
         await this.processBatch(batch);
         await this.updateProgress();
         await this.checkRateLimits();
       }
       
       // Post-process
       await this.reconcile();
       return this.generateReport();
     }
   }

4. **Progress Tracking**

   class ProgressTracker {
     constructor(total) {
       this.total = total;
       this.completed = 0;
       this.failed = 0;
       this.startTime = Date.now();
     }
     
     update(success = true) {
       success ? this.completed++ : this.failed++;
       this.render();
     }
     
     render() {
       const progress = (this.completed + this.failed) / this.total;
       const elapsed = Date.now() - this.startTime;
       const eta = (elapsed / progress) - elapsed;
       
       console.log(`
   Importing GitHub Issues to Linear
   ════════════════════════════════
   
   Progress: [${'█'.repeat(progress * 30)}${' '.repeat(30 - progress * 30)}] ${(progress * 100).toFixed(1)}%
   
   Completed: ${this.completed}/${this.total}
   Failed: ${this.failed}
   Rate: ${(this.completed / (elapsed / 1000)).toFixed(1)} issues/sec
   ETA: ${formatTime(eta)}
   
   Current: ${this.currentItem?.title || 'Processing...'}
       `);
     }
   }

5. **Error Handling**

   async function handleImportError(issue, error, attempt) {
     const errorType = classifyError(error);
     
     switch (errorType) {
       case 'RATE_LIMIT':
         await waitForRateLimit(error);
         return 'RETRY';
         
       case 'DUPLICATE':
         logDuplicate(issue);
         return 'SKIP';
         
       case 'VALIDATION':
         const fixed = await tryAutoFix(issue, error);
         return fixed ? 'RETRY' : 'FAIL';
         
       case 'NETWORK':
         if (attempt < BATCH_CONFIG.retryAttempts) {
           await exponentialBackoff(attempt);
           return 'RETRY';
         }
         return 'FAIL';
         
       default:
         return 'FAIL';
     }
   }

6. **Data Transformation**

   async function transformIssuesBatch(issues) {
     return Promise.all(issues.map(async issue => {
       try {
         return {
           title: sanitizeTitle(issue.title),
           description: await enhanceDescription(issue),
           priority: calculatePriority(issue),
           state: mapState(issue.state),
           labels: await mapLabels(issue.labels),
           assignee: await findLinearUser(issue.assignee),
           
           metadata: {
             githubNumber: issue.number,
             githubUrl: issue.html_url,
             importedAt: new Date().toISOString(),
             importBatch: this.batchId
           }
         };
       } catch (error) {
         return { error, issue };
       }
     }));
   }

7. **Duplicate Detection**

   async function checkDuplicates(issues) {
     const existingTasks = await linear.issues({
       filter: { 
         externalId: { in: issues.map(i => `gh-${i.number}`) }
       }
     });
     
     const duplicates = new Map();
     for (const task of existingTasks) {
       duplicates.set(task.externalId, task);
     }
     
     return {
       hasDuplicates: duplicates.size > 0,
       duplicates: duplicates,
       unique: issues.filter(i => !duplicates.has(`gh-${i.number}`))
     };
   }

8. **Rate Limit Management**

   class RateLimitManager {
     constructor() {
       this.github = { limit: 5000, remaining: 5000, reset: null };
       this.linear = { limit: 1500, remaining: 1500, reset: null };
     }
     
     async checkAndWait() {
       // Update current limits
       await this.updateLimits();
       
       // GitHub check
       if (this.github.remaining < 100) {