Technical Insights & API Development

// Example API endpoint structure
GET /api/v1/parsers/encar/vehicles
GET /api/v1/parsers/encar/vehicles/{id}
GET /api/v1/parsers/encar/vehicles/{id}/specifications
GET /api/v1/parsers/encar/vehicles/{id}/pricing

{
  "success": true,
  "data": {
    "vehicles": [
      {
        "id": "encar_12345",
        "make": "Hyundai",
        "model": "Tucson",
        "year": 2023,
        "price": 25000,
        "specifications": {
          "engine": "2.0L Turbo",
          "transmission": "Automatic",
          "fuel_type": "Gasoline"
        }
      }
    ]
  },
  "pagination": {
    "page": 1,
    "limit": 20,
    "total": 1500
  }
}

interface APIError {
  code: string;
  message: string;
  details?: any;
  timestamp: string;
}

// Error response example
{
  "success": false,
  "error": {
    "code": "RATE_LIMIT_EXCEEDED",
    "message": "Rate limit exceeded. Please try again in 60 seconds.",
    "timestamp": "2024-01-15T10:30:00Z"
  }
}

# Example anti-detection configuration
config = {
    "user_agents": [
        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
        "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36"
    ],
    "proxy_rotation": {
        "enabled": True,
        "interval": 300,  # 5 minutes
        "pool_size": 100
    },
    "request_delays": {
        "min": 1.0,
        "max": 3.0
    },
    "session_management": {
        "cookies_enabled": True,
        "session_persistence": True
    }
}

// Puppeteer configuration for automotive sites
const browserConfig = {
    headless: true,
    args: [
        '--no-sandbox',
        '--disable-setuid-sandbox',
        '--disable-dev-shm-usage',
        '--disable-accelerated-2d-canvas',
        '--no-first-run',
        '--no-zygote',
        '--disable-gpu'
    ],
    userAgent: 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
};

# Data processing pipeline
class AutomotiveDataProcessor:
    def __init__(self):
        self.validators = [
            PriceValidator(),
            SpecificationValidator(),
            ImageValidator()
        ]
    
    def process_vehicle_data(self, raw_data):
        # Clean and validate data
        cleaned_data = self.clean_data(raw_data)
        
        # Apply business rules
        processed_data = self.apply_business_rules(cleaned_data)
        
        # Validate final output
        validated_data = self.validate_data(processed_data)
        
        return validated_data

// Redis caching implementation
class CacheManager {
    private redis: Redis;
    
    async getCachedData(key: string): Promise<any> {
        const cached = await this.redis.get(key);
        if (cached) {
            return JSON.parse(cached);
        }
        return null;
    }
    
    async setCachedData(key: string, data: any, ttl: number = 3600): Promise<void> {
        await this.redis.setex(key, ttl, JSON.stringify(data));
    }
}

-- Optimized queries for automotive data
CREATE INDEX idx_vehicle_make_model ON vehicles(make, model);
CREATE INDEX idx_vehicle_price ON vehicles(price);
CREATE INDEX idx_vehicle_year ON vehicles(year);

-- Partitioned tables for large datasets
CREATE TABLE vehicle_data_2024 PARTITION OF vehicle_data
FOR VALUES FROM ('2024-01-01') TO ('2025-01-01');

# Docker Compose configuration
version: '3.8'
services:
  api-server:
    image: carapis/api:latest
    deploy:
      replicas: 5
      resources:
        limits:
          cpus: '1.0'
          memory: 1G
    environment:
      - NODE_ENV=production
      - REDIS_URL=redis://redis:6379

// JWT-based authentication
interface AuthConfig {
    secret: string;
    expiresIn: string;
    refreshExpiresIn: string;
}

class AuthService {
    generateToken(user: User): string {
        return jwt.sign(
            { userId: user.id, role: user.role },
            process.env.JWT_SECRET,
            { expiresIn: '1h' }
        );
    }
    
    verifyToken(token: string): DecodedToken {
        return jwt.verify(token, process.env.JWT_SECRET);
    }
}

// Rate limiting implementation
import rateLimit from 'express-rate-limit';

const apiLimiter = rateLimit({
    windowMs: 15 * 60 * 1000, // 15 minutes
    max: 100, // limit each IP to 100 requests per windowMs
    message: 'Too many requests from this IP, please try again later.',
    standardHeaders: true,
    legacyHeaders: false,
});

// Data encryption for sensitive information
class DataEncryption {
    private algorithm = 'aes-256-gcm';
    private key = process.env.ENCRYPTION_KEY;
    
    encrypt(data: string): string {
        const iv = crypto.randomBytes(16);
        const cipher = crypto.createCipher(this.algorithm, this.key);
        let encrypted = cipher.update(data, 'utf8', 'hex');
        encrypted += cipher.final('hex');
        return iv.toString('hex') + ':' + encrypted;
    }
    
    decrypt(encryptedData: string): string {
        const [ivHex, encrypted] = encryptedData.split(':');
        const iv = Buffer.from(ivHex, 'hex');
        const decipher = crypto.createDecipher(this.algorithm, this.key);
        let decrypted = decipher.update(encrypted, 'hex', 'utf8');
        decrypted += decipher.final('utf8');
        return decrypted;
    }
}

// Performance monitoring setup
class PerformanceMonitor {
    trackRequest(req: Request, res: Response, next: NextFunction) {
        const start = Date.now();
        
        res.on('finish', () => {
            const duration = Date.now() - start;
            const status = res.statusCode;
            
            // Log metrics
            this.logMetrics({
                endpoint: req.path,
                method: req.method,
                duration,
                status,
                timestamp: new Date()
            });
        });
        
        next();
    }
    
    private logMetrics(metrics: RequestMetrics) {
        // Send to monitoring service
        console.log('API Metrics:', metrics);
    }
}

// Error tracking implementation
class ErrorTracker {
    trackError(error: Error, context: ErrorContext) {
        const errorReport = {
            message: error.message,
            stack: error.stack,
            context,
            timestamp: new Date(),
            environment: process.env.NODE_ENV
        };
        
        // Send to error tracking service
        this.sendToErrorService(errorReport);
    }
}

// API testing with Jest
describe('Automotive Data API', () => {
    test('should return vehicle data', async () => {
        const response = await request(app)
            .get('/api/v1/parsers/encar/vehicles')
            .set('Authorization', `Bearer ${token}`)
            .expect(200);
        
        expect(response.body.success).toBe(true);
        expect(response.body.data.vehicles).toBeDefined();
    });
    
    test('should handle rate limiting', async () => {
        // Make multiple requests to trigger rate limit
        for (let i = 0; i < 101; i++) {
            await request(app)
                .get('/api/v1/parsers/encar/vehicles')
                .set('Authorization', `Bearer ${token}`);
        }
        
        const response = await request(app)
            .get('/api/v1/parsers/encar/vehicles')
            .set('Authorization', `Bearer ${token}`)
            .expect(429);
        
        expect(response.body.error.code).toBe('RATE_LIMIT_EXCEEDED');
    });
});

// Load testing with Artillery
import { Artillery } from 'artillery';

const config = {
    target: 'https://api.carapis.com',
    phases: [
        { duration: 60, arrivalRate: 10 },
        { duration: 120, arrivalRate: 50 },
        { duration: 60, arrivalRate: 100 }
    ],
    scenarios: [
        {
            name: 'Vehicle Data API',
            requests: [
                {
                    method: 'GET',
                    url: '/api/v1/parsers/encar/vehicles'
                }
            ]
        }
    ]
};

# Dockerfile for automotive API
FROM node:18-alpine

WORKDIR /app

COPY package*.json ./
RUN npm ci --only=production

COPY dist/ ./dist/
COPY .env.production ./

EXPOSE 3000

CMD ["npm", "start"]

# GitHub Actions workflow
name: Deploy Automotive API

on:
  push:
    branches: [main]

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-node@v3
        with:
          node-version: '18'
      - run: npm ci
      - run: npm test
      - run: npm run build

  deploy:
    needs: test
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Deploy to production
        run: |
          docker build -t carapis/api .
          docker push carapis/api:latest

# ML-powered data validation
class MLDataValidator:
    def __init__(self):
        self.model = self.load_validation_model()
    
    def validate_vehicle_data(self, data):
        features = self.extract_features(data)
        prediction = self.model.predict(features)
        return prediction > 0.8  # 80% confidence threshold

// Real-time data streaming
class RealTimeProcessor {
    private kafka = new Kafka();
    
    async processStreamingData(topic: string) {
        const consumer = this.kafka.consumer({ groupId: 'automotive-data' });
        await consumer.subscribe({ topic });
        
        await consumer.run({
            eachMessage: async ({ topic, partition, message }) => {
                const data = JSON.parse(message.value.toString());
                await this.processVehicleData(data);
            },
        });
    }
}