// Advanced replica set configuration
rs.initiate({
  _id: "production-replica-set",
  version: 1,
  members: [
    {
      _id: 0,
      host: "mongo-primary.internal:27017",
      priority: 2,
      tags: { region: "us-east-1", role: "primary" }
    },
    {
      _id: 1,
      host: "mongo-secondary-1.internal:27017",
      priority: 1,
      tags: { region: "us-east-1", role: "secondary" }
    },
    {
      _id: 2,
      host: "mongo-secondary-2.internal:27017",
      priority: 1,
      tags: { region: "us-west-2", role: "secondary" }
    },
    {
      _id: 3,
      host: "mongo-analytics.internal:27017",
      priority: 0,
      hidden: true,
      votes: 0,
      tags: { role: "analytics", workload: "reporting" }
    },
    {
      _id: 4,
      host: "mongo-delayed.internal:27017",
      priority: 0,
      hidden: true,
      votes: 0,
      slaveDelay: 3600, // 1-hour delay for disaster recovery
      tags: { role: "delayed-secondary" }
    }
  ],
  settings: {
    chainingAllowed: true,
    heartbeatIntervalMillis: 2000,
    heartbeatTimeoutSecs: 10,
    electionTimeoutMillis: 10000,
    catchUpTimeoutMillis: 60000,
    getLastErrorModes: {
      "cross-region": { region: 2 },
      "majority-plus-analytics": { role: 3 }
    },
    getLastErrorDefaults: {
      w: "majority",
      wtimeout: 5000
    }
  }
});

// Configure read preferences for different workloads
db.getMongo().setReadPref("primaryPreferred", [
  { region: "us-east-1" }
]);

// Analytics workload configuration
db.getMongo().setReadPref("secondary", [
  { role: "analytics" }
]);

// Write concern strategies for different use cases
const writeConfigs = {
  // Critical financial transactions
  financial: {
    w: "majority",
    j: true,
    wtimeout: 5000
  },
  
  // Cross-region durability
  crossRegion: {
    w: "cross-region",
    j: true,
    wtimeout: 10000
  },
  
  // High-throughput logging
  logging: {
    w: 1,
    j: false,
    wtimeout: 1000
  },
  
  // Analytics data ingestion
  analytics: {
    w: "majority-plus-analytics",
    j: true,
    wtimeout: 15000
  }
};

// Application-specific implementations
await db.transactions.insertOne(
  { 
    userId: ObjectId("..."),
    amount: 1000.00,
    type: "transfer",
    timestamp: new Date()
  },
  { writeConcern: writeConfigs.financial }
);

// Read preference optimization
const readPreferences = {
  // Real-time user data
  userProfile: { mode: "primary" },
  
  // Analytics and reporting
  analytics: { 
    mode: "secondary",
    tags: [{ role: "analytics" }],
    maxStalenessSeconds: 300
  },
  
  // Search and discovery
  search: {
    mode: "secondaryPreferred",
    tags: [{ region: "us-east-1" }]
  }
};

// Anti-patterns to avoid
const badShardKeys = {
  // Monotonically increasing - creates hotspots
  timestamp: { _id: 1 },
  autoIncrement: { sequence: 1 },
  
  // Low cardinality - limits scaling
  status: { status: 1 },
  category: { category: 1 },
  
  // Non-queryable - forces scatter-gather
  hash: { _id: "hashed" } // without query patterns
};

// Effective shard key strategies
const goodShardKeys = {
  // High cardinality + query pattern alignment
  userActivity: { userId: 1, timestamp: 1 },
  
  // Compound key for even distribution
  ecommerce: { customerId: 1, orderId: 1 },
  
  // Geographic distribution
  iot: { deviceRegion: 1, deviceId: 1, timestamp: 1 },
  
  // Hash for even distribution when range isn't suitable
  socialMedia: { userId: "hashed" }
};

// Implementation with zone sharding
sh.enableSharding("ecommerce");

// Create zones for geographic distribution
sh.addShardToZone("shard-us-east", "us-east");
sh.addShardToZone("shard-us-west", "us-west");
sh.addShardToZone("shard-europe", "europe");

// Define zone ranges
sh.updateZoneKeyRange(
  "ecommerce.orders",
  { region: "us-east", customerId: MinKey },
  { region: "us-east", customerId: MaxKey },
  "us-east"
);

// Shard the collection
sh.shardCollection("ecommerce.orders", { 
  region: 1, 
  customerId: 1, 
  orderDate: 1 
});

// Custom balancer settings for different workloads
db.adminCommand({
  balancerCollectionStatus: "ecommerce.orders",
  defragmentCollection: true
});

// Configure balancer windows
db.settings.updateOne(
  { _id: "balancer" },
  { 
    $set: { 
      activeWindow: {
        start: "02:00",  // 2 AM
        stop: "06:00"    // 6 AM
      },
      mode: "full"
    }
  },
  { upsert: true }
);

// Optimize chunk size for specific collections
db.adminCommand({
  configureCollectionBalancing: "ecommerce.orders",
  chunkSize: 128,  // MB
  defragmentCollection: true,
  enableAutoSplit: true
});

// Schema versioning for backward compatibility
const userSchemaV1 = {
  _id: ObjectId("..."),
  schemaVersion: 1,
  email: "user@example.com",
  name: "John Doe",
  created: new Date()
};

const userSchemaV2 = {
  _id: ObjectId("..."),
  schemaVersion: 2,
  email: "user@example.com",
  profile: {
    firstName: "John",
    lastName: "Doe",
    displayName: "John Doe"
  },
  preferences: {
    theme: "dark",
    notifications: true,
    language: "en"
  },
  created: new Date(),
  lastModified: new Date()
};

// Migration strategy with version checks
function migrateUser(user) {
  switch (user.schemaVersion || 1) {
    case 1:
      return {
        ...user,
        schemaVersion: 2,
        profile: {
          firstName: user.name.split(' ')[0],
          lastName: user.name.split(' ')[1] || '',
          displayName: user.name
        },
        preferences: {
          theme: "light",
          notifications: true,
          language: "en"
        },
        lastModified: new Date()
      };
    case 2:
      return user; // Already current version
    default:
      throw new Error(`Unknown schema version: ${user.schemaVersion}`);
  }
}

// Product catalog with complex relationships
const productCatalog = {
  // Embedded for frequently accessed, bounded data
  product: {
    _id: ObjectId("..."),
    sku: "LAPTOP-001",
    name: "Gaming Laptop",
    
    // Embedded specifications (1-to-1, rarely changes)
    specifications: {
      processor: "Intel i7-12700H",
      memory: "32GB DDR4",
      storage: "1TB NVMe SSD",
      graphics: "RTX 3070",
      display: {
        size: "15.6 inches",
        resolution: "1920x1080",
        refreshRate: "144Hz"
      }
    },
    
    // Embedded pricing (frequently accessed with product)
    pricing: {
      msrp: 1499.99,
      currentPrice: 1299.99,
      currency: "USD",
      discounts: [
        {
          type: "seasonal",
          amount: 200.00,
          validUntil: new Date("2024-12-31")
        }
      ]
    },
    
    // Referenced for large, independently managed data
    categoryId: ObjectId("category-laptops"),
    brandId: ObjectId("brand-gaming"),
    
    // Embedded recent reviews (limited subset)
    recentReviews: [
      {
        _id: ObjectId("..."),
        userId: ObjectId("..."),
        rating: 5,
        title: "Excellent performance",
        summary: "Great for gaming and development work...",
        date: new Date(),
        helpful: 23
      }
    ],
    
    // Referenced for complete review data
    totalReviews: 156,
    averageRating: 4.7,
    
    // Embedded inventory (frequently updated)
    inventory: {
      available: 45,
      reserved: 12,
      reorderLevel: 10,
      lastUpdated: new Date()
    }
  }
};

// Separate collections for scalable data
const reviews = {
  _id: ObjectId("..."),
  productId: ObjectId("product-id"),
  userId: ObjectId("user-id"),
  rating: 5,
  title: "Excellent performance",
  content: "Detailed review content...",
  images: ["review-img-1.jpg", "review-img-2.jpg"],
  verified: true,
  helpful: 23,
  replies: [
    {
      userId: ObjectId("..."),
      content: "Thanks for the detailed review!",
      date: new Date()
    }
  ],
  created: new Date(),
  lastModified: new Date()
};

// Event-driven architecture with polymorphic events
const eventStore = [
  {
    _id: ObjectId("..."),
    eventType: "UserRegistered",
    aggregateId: "user-123",
    aggregateVersion: 1,
    eventData: {
      email: "user@example.com",
      registrationSource: "web",
      marketingConsent: true
    },
    metadata: {
      ipAddress: "192.168.1.1",
      userAgent: "Mozilla/5.0...",
      correlationId: "reg-123"
    },
    timestamp: new Date(),
    processed: false
  },
  {
    _id: ObjectId("..."),
    eventType: "OrderPlaced",
    aggregateId: "order-456",
    aggregateVersion: 1,
    eventData: {
      customerId: "user-123",
      items: [
        { productId: "prod-1", quantity: 2, price: 29.99 },
        { productId: "prod-2", quantity: 1, price: 15.50 }
      ],
      totalAmount: 75.48,
      paymentMethod: "stripe",
      shippingAddress: {
        street: "123 Main St",
        city: "New York",
        state: "NY",
        zip: "10001"
      }
    },
    metadata: {
      sessionId: "sess-789",
      promotionCode: "SAVE10"
    },
    timestamp: new Date(),
    processed: false
  }
];

// Polymorphic query patterns
const eventHandlers = {
  async handleUserRegistered(event) {
    await db.users.insertOne({
      _id: event.aggregateId,
      email: event.eventData.email,
      registrationSource: event.eventData.registrationSource,
      marketingConsent: event.eventData.marketingConsent,
      created: event.timestamp
    });
  },
  
  async handleOrderPlaced(event) {
    await db.orders.insertOne({
      _id: event.aggregateId,
      customerId: event.eventData.customerId,
      items: event.eventData.items,
      totalAmount: event.eventData.totalAmount,
      status: "pending",
      created: event.timestamp
    });
  }
};

// Event processing with polymorphic dispatch
async function processEvents() {
  const unprocessedEvents = await db.events.find({ processed: false })
    .sort({ timestamp: 1 })
    .limit(100)
    .toArray();
  
  for (const event of unprocessedEvents) {
    const handlerName = `handle${event.eventType}`;
    const handler = eventHandlers[handlerName];
    
    if (handler) {
      try {
        await handler(event);
        await db.events.updateOne(
          { _id: event._id },
          { $set: { processed: true, processedAt: new Date() } }
        );
      } catch (error) {
        await db.events.updateOne(
          { _id: event._id },
          { 
            $set: { 
              processingError: error.message,
              processingAttempts: (event.processingAttempts || 0) + 1
            }
          }
        );
      }
    }
  }
}

// Time series collection for IoT sensor data
db.createCollection("sensor_data", {
  timeseries: {
    timeField: "timestamp",
    metaField: "sensor",
    granularity: "minutes",
    bucketMaxSpanSeconds: 3600,
    bucketRoundingSeconds: 60
  },
  expireAfterSeconds: 31536000 // 1 year retention
});

// Optimized IoT data structure
const sensorReading = {
  timestamp: new Date(),
  sensor: {
    deviceId: "sensor-001",
    location: "warehouse-a",
    type: "temperature-humidity",
    model: "DHT22"
  },
  measurements: {
    temperature: 23.5,
    humidity: 65.2,
    batteryLevel: 87
  },
  metadata: {
    firmware: "v1.2.3",
    signalStrength: -45,
    dataQuality: "good"
  }
};

// Efficient time series queries
const temperatureAggregation = [
  {
    $match: {
      "timestamp": {
        $gte: new Date(Date.now() - 24 * 60 * 60 * 1000) // Last 24 hours
      },
      "sensor.type": "temperature-humidity"
    }
  },
  {
    $group: {
      _id: {
        deviceId: "$sensor.deviceId",
        hour: {
          $dateTrunc: {
            date: "$timestamp",
            unit: "hour"
          }
        }
      },
      avgTemperature: { $avg: "$measurements.temperature" },
      maxTemperature: { $max: "$measurements.temperature" },
      minTemperature: { $min: "$measurements.temperature" },
      readingCount: { $sum: 1 }
    }
  },
  {
    $sort: { "_id.hour": 1 }
  }
];

// Real-time anomaly detection
const anomalyDetection = [
  {
    $match: {
      "timestamp": {
        $gte: new Date(Date.now() - 60 * 60 * 1000) // Last hour
      }
    }
  },
  {
    $setWindowFields: {
      partitionBy: "$sensor.deviceId",
      sortBy: { timestamp: 1 },
      output: {
        movingAverage: {
          $avg: "$measurements.temperature",
          window: {
            documents: [-10, 0] // 10-point moving average
          }
        },
        standardDeviation: {
          $stdDevSamp: "$measurements.temperature",
          window: {
            documents: [-20, 0]
          }
        }
      }
    }
  },
  {
    $addFields: {
      anomaly: {
        $gt: [
          {
            $abs: {
              $subtract: ["$measurements.temperature", "$movingAverage"]
            }
          },
          { $multiply: ["$standardDeviation", 2] } // 2-sigma threshold
        ]
      }
    }
  },
  {
    $match: { anomaly: true }
  }
];

// Query pattern analysis for index design
const queryPatterns = {
  // User activity lookup
  userActivity: {
    query: { userId: ObjectId("..."), timestamp: { $gte: new Date() } },
    sort: { timestamp: -1 },
    limit: 50
  },
  
  // Product search with filters
  productSearch: {
    query: { 
      category: "electronics", 
      price: { $gte: 100, $lte: 500 },
      inStock: true 
    },
    sort: { popularity: -1, price: 1 }
  },
  
  // Geospatial queries
  nearbyStores: {
    query: {
      location: {
        $near: {
          $geometry: { type: "Point", coordinates: [-73.9857, 40.7484] },
          $maxDistance: 5000
        }
      },
      storeType: "retail"
    }
  }
};

// Optimized index strategies
const indexStrategies = {
  // ESR (Equality, Sort, Range) principle
  userActivityIndex: {
    keys: { userId: 1, timestamp: -1 }, // Equality first, then sort
    options: { 
      name: "idx_user_activity_optimized",
      partialFilterExpression: { timestamp: { $exists: true } }
    }
  },
  
  // Compound index for product filtering
  productSearchIndex: {
    keys: { category: 1, inStock: 1, price: 1, popularity: -1 },
    options: {
      name: "idx_product_search_compound",
      partialFilterExpression: { inStock: true }
    }
  },
  
  // Geospatial index
  storeLocationIndex: {
    keys: { location: "2dsphere", storeType: 1 },
    options: { name: "idx_store_location_type" }
  },
  
  // Text search with language support
  productTextSearch: {
    keys: { 
      name: "text", 
      description: "text", 
      tags: "text" 
    },
    options: {
      name: "idx_product_text_search",
      weights: { name: 10, description: 5, tags: 1 },
      default_language: "english"
    }
  }
};

// Index creation with monitoring
async function createOptimizedIndexes() {
  const collections = [
    { name: "user_activity", indexes: [indexStrategies.userActivityIndex] },
    { name: "products", indexes: [
      indexStrategies.productSearchIndex,
      indexStrategies.productTextSearch
    ]},
    { name: "stores", indexes: [indexStrategies.storeLocationIndex] }
  ];
  
  for (const collection of collections) {
    for (const index of collection.indexes) {
      try {
        const result = await db[collection.name].createIndex(
          index.keys, 
          index.options
        );
        console.log(`Created index ${index.options.name}: ${result}`);
      } catch (error) {
        console.error(`Failed to create index ${index.options.name}:`, error);
      }
    }
  }
}

// Sparse indexes for optional fields
db.users.createIndex(
  { socialSecurityNumber: 1 },
  { 
    sparse: true,
    name: "idx_users_ssn_sparse",
    unique: true
  }
);

// Partial indexes for specific conditions
db.orders.createIndex(
  { customerId: 1, orderDate: -1 },
  {
    partialFilterExpression: { 
      status: { $in: ["pending", "processing"] }
    },
    name: "idx_orders_active_customer_date"
  }
);

// TTL indexes for automatic document expiration
db.sessions.createIndex(
  { lastAccessed: 1 },
  { expireAfterSeconds: 3600 } // 1 hour
);

db.logs.createIndex(
  { createdAt: 1 },
  { expireAfterSeconds: 2592000 } // 30 days
);

// Compound TTL with partial filter
db.temporaryData.createIndex(
  { expiresAt: 1 },
  {
    expireAfterSeconds: 0,
    partialFilterExpression: { 
      temporary: true 
    }
  }
);

// Inefficient aggregation pipeline
const inefficientPipeline = [
  { $match: { status: "completed" } },
  { $lookup: {
      from: "products",
      localField: "productId",
      foreignField: "_id",
      as: "product"
    }
  },
  { $unwind: "$product" },
  { $match: { "product.category": "electronics" } }, // Should be earlier
  { $group: {
      _id: "$customerId",
      totalSpent: { $sum: "$amount" }
    }
  },
  { $sort: { totalSpent: -1 } },
  { $limit: 100 }
];

// Optimized aggregation pipeline
const optimizedPipeline = [
  // Early filtering reduces documents in pipeline
  { $match: { 
      status: "completed",
      amount: { $gt: 0 } // Additional early filter
    }
  },
  
  // Index hint for optimal query plan
  { $hint: "idx_orders_status_customer_amount" },
  
  // Efficient lookup with pipeline for filtering
  { $lookup: {
      from: "products",
      let: { productId: "$productId" },
      pipeline: [
        { $match: { 
            $expr: { $eq: ["$_id", "$$productId"] },
            category: "electronics" // Filter in subpipeline
          }
        },
        { $project: { name: 1, category: 1 } } // Minimal projection
      ],
      as: "product"
    }
  },
  
  // Filter out documents without matching products
  { $match: { "product.0": { $exists: true } } },
  
  // Group with optimized accumulator
  { $group: {
      _id: "$customerId",
      totalSpent: { $sum: "$amount" },
      orderCount: { $sum: 1 },
      avgOrderValue: { $avg: "$amount" }
    }
  },
  
  // Sort and limit for top customers
  { $sort: { totalSpent: -1 } },
  { $limit: 100 },
  
  // Final projection for clean output
  { $project: {
      customerId: "$_id",
      totalSpent: 1,
      orderCount: 1,
      avgOrderValue: { $round: ["$avgOrderValue", 2] },
      _id: 0
    }
  }
];

// Performance monitoring wrapper
async function executeOptimizedAggregation(pipeline, options = {}) {
  const startTime = Date.now();
  
  try {
    const result = await db.orders.aggregate(pipeline, {
      allowDiskUse: true,
      cursor: { batchSize: 1000 },
      ...options
    }).toArray();
    
    const executionTime = Date.now() - startTime;
    
    // Log performance metrics
    await db.query_performance.insertOne({
      query: "customer_electronics_analysis",
      executionTime,
      resultCount: result.length,
      timestamp: new Date(),
      pipeline: pipeline.length
    });
    
    return result;
  } catch (error) {
    console.error("Aggregation failed:", error);
    throw error;
  }
}

// Query performance analysis
async function analyzeQueryPerformance() {
  const query = { 
    category: "electronics", 
    price: { $gte: 100, $lte: 1000 },
    inStock: true 
  };
  
  // Explain query execution
  const explanation = await db.products.find(query)
    .sort({ popularity: -1 })
    .limit(20)
    .explain("executionStats");
  
  console.log("Query Execution Stats:");
  console.log(`Execution Time: ${explanation.executionStats.executionTimeMillis}ms`);
  console.log(`Documents Examined: ${explanation.executionStats.totalDocsExamined}`);
  console.log(`Documents Returned: ${explanation.executionStats.totalDocsReturned}`);
  console.log(`Index Used: ${explanation.executionStats.executionStages.indexName || 'COLLSCAN'}`);
  
  // Calculate selectivity
  const selectivity = explanation.executionStats.totalDocsReturned / 
                     explanation.executionStats.totalDocsExamined;
  
  if (selectivity < 0.1) {
    console.warn("Low selectivity detected. Consider index optimization.");
  }
  
  return explanation;
}

// Index usage monitoring
async function monitorIndexUsage() {
  const indexStats = await db.products.aggregate([
    { $indexStats: {} }
  ]).toArray();
  
  console.log("Index Usage Statistics:");
  indexStats.forEach(stat => {
    console.log(`Index: ${stat.name}`);
    console.log(`  Accesses: ${stat.accesses.ops}`);
    console.log(`  Since: ${stat.accesses.since}`);
  });
  
  // Identify unused indexes
  const unusedIndexes = indexStats.filter(stat => stat.accesses.ops === 0);
  if (unusedIndexes.length > 0) {
    console.warn("Unused indexes detected:", unusedIndexes.map(idx => idx.name));
  }
}

// User Service - Event Store Implementation
class UserEventStore {
  constructor(db) {
    this.db = db;
    this.collection = db.collection('user_events');
  }
  
  async appendEvent(streamId, expectedVersion, events) {
    const session = this.db.client.startSession();
    
    try {
      await session.withTransaction(async () => {
        // Check current version for optimistic concurrency
        const currentEvents = await this.collection
          .find({ streamId }, { session })
          .sort({ version: -1 })
          .limit(1)
          .toArray();
        
        const currentVersion = currentEvents.length > 0 ? 
          currentEvents[0].version : 0;
        
        if (currentVersion !== expectedVersion) {
          throw new Error(`Concurrency conflict. Expected ${expectedVersion}, got ${currentVersion}`);
        }
        
        // Append new events
        const eventsToInsert = events.map((event, index) => ({
          _id: new ObjectId(),
          streamId,
          version: expectedVersion + index + 1,
          eventType: event.type,
          eventData: event.data,
          metadata: {
            ...event.metadata,
            timestamp: new Date(),
            correlationId: event.metadata?.correlationId || new ObjectId().toString()
          }
        }));
        
        await this.collection.insertMany(eventsToInsert, { session });
        
        return eventsToInsert;
      });
    } finally {
      await session.endSession();
    }
  }
  
  async getEvents(streamId, fromVersion = 0) {
    return await this.collection
      .find({ 
        streamId, 
        version: { $gt: fromVersion } 
      })
      .sort({ version: 1 })
      .toArray();
  }
}

// Saga Orchestration Pattern
class OrderSagaOrchestrator {
  constructor(db) {
    this.db = db;
    this.sagaCollection = db.collection('order_sagas');
    this.stepCollection = db.collection('saga_steps');
  }
  
  async startOrderSaga(orderData) {
    const sagaId = new ObjectId();
    const saga = {
      _id: sagaId,
      sagaType: 'ProcessOrder',
      state: 'Started',
      currentStep: 0,
      data: orderData,
      steps: [
        { name: 'ReserveInventory', status: 'Pending' },
        { name: 'ProcessPayment', status: 'Pending' },
        { name: 'CreateShipment', status: 'Pending' },
        { name: 'SendConfirmation', status: 'Pending' }
      ],
      createdAt: new Date(),
      updatedAt: new Date()
    };
    
    await this.sagaCollection.insertOne(saga);
    await this.executeNextStep(sagaId);
    
    return sagaId;
  }
  
  async executeNextStep(sagaId) {
    const saga = await this.sagaCollection.findOne({ _id: sagaId });
    if (!saga || saga.currentStep >= saga.steps.length) {
      return;
    }
    
    const currentStep = saga.steps[saga.currentStep];
    
    try {
      // Execute step based on type
      const result = await this.executeStep(currentStep.name, saga.data);
      
      // Update step status
      await this.sagaCollection.updateOne(
        { _id: sagaId },
        {
          $set: {
            [`steps.${saga.currentStep}.status`]: 'Completed',
            [`steps.${saga.currentStep}.result`]: result,
            currentStep: saga.currentStep + 1,
            updatedAt: new Date()
          }
        }
      );
      
      // Continue to next step
      await this.executeNextStep(sagaId);
      
    } catch (error) {
      // Handle step failure
      await this.handleStepFailure(sagaId, saga.currentStep, error);
    }
  }
  
  async handleStepFailure(sagaId, failedStepIndex, error) {
    // Mark step as failed
    await this.sagaCollection.updateOne(
      { _id: sagaId },
      {
        $set: {
          [`steps.${failedStepIndex}.status`]: 'Failed',
          [`steps.${failedStepIndex}.error`]: error.message,
          state: 'Compensating',
          updatedAt: new Date()
        }
      }
    );
    
    // Execute compensation actions
    await this.executeCompensation(sagaId, failedStepIndex);
  }
  
  async executeCompensation(sagaId, fromStep) {
    const saga = await this.sagaCollection.findOne({ _id: sagaId });
    
    // Compensate completed steps in reverse order
    for (let i = fromStep - 1; i >= 0; i--) {
      const step = saga.steps[i];
      if (step.status === 'Completed') {
        try {
          await this.compensateStep(step.name, step.result, saga.data);
          
          await this.sagaCollection.updateOne(
            { _id: sagaId },
            {
              $set: {
                [`steps.${i}.compensated`]: true,
                updatedAt: new Date()
              }
            }
          );
        } catch (compensationError) {
          console.error(`Compensation failed for step ${step.name}:`, compensationError);
          // Handle compensation failure (manual intervention required)
        }
      }
    }
    
    await this.sagaCollection.updateOne(
      { _id: sagaId },
      { $set: { state: 'Failed', updatedAt: new Date() } }
    );
  }
}

// Command Side - Write Model
class OrderCommandHandler {
  constructor(eventStore, db) {
    this.eventStore = eventStore;
    this.db = db;
  }
  
  async createOrder(command) {
    const orderId = new ObjectId().toString();
    const events = [
      {
        type: 'OrderCreated',
        data: {
          orderId,
          customerId: command.customerId,
          items: command.items,
          totalAmount: command.totalAmount
        },
        metadata: {
          commandId: command.id,
          userId: command.userId,
          timestamp: new Date()
        }
      }
    ];
    
    await this.eventStore.appendEvent(`order-${orderId}`, 0, events);
    return orderId;
  }
  
  async updateOrderStatus(command) {
    const events = await this.eventStore.getEvents(`order-${command.orderId}`);
    const order = this.reconstructOrderFromEvents(events);
    
    if (!order) {
      throw new Error('Order not found');
    }
    
    const newEvent = {
      type: 'OrderStatusChanged',
      data: {
        orderId: command.orderId,
        previousStatus: order.status,
        newStatus: command.newStatus,
        reason: command.reason
      },
      metadata: {
        commandId: command.id,
        userId: command.userId,
        timestamp: new Date()
      }
    };
    
    await this.eventStore.appendEvent(
      `order-${command.orderId}`, 
      events.length, 
      [newEvent]
    );
  }
  
  reconstructOrderFromEvents(events) {
    let order = null;
    
    for (const event of events) {
      switch (event.eventType) {
        case 'OrderCreated':
          order = {
            id: event.eventData.orderId,
            customerId: event.eventData.customerId,
            items: event.eventData.items,
            totalAmount: event.eventData.totalAmount,
            status: 'Created',
            createdAt: event.metadata.timestamp
          };
          break;
          
        case 'OrderStatusChanged':
          if (order) {
            order.status = event.eventData.newStatus;
            order.lastModified = event.metadata.timestamp;
          }
          break;
      }
    }
    
    return order;
  }
}

// Query Side - Read Model Projections
class OrderProjectionHandler {
  constructor(db) {
    this.db = db;
    this.orderSummaryCollection = db.collection('order_summary');
    this.customerOrdersCollection = db.collection('customer_orders');
  }
  
  async handleOrderCreated(event) {
    const orderSummary = {
      _id: event.eventData.orderId,
      customerId: event.eventData.customerId,
      itemCount: event.eventData.items.length,
      totalAmount: event.eventData.totalAmount,
      status: 'Created',
      createdAt: event.metadata.timestamp,
      lastModified: event.metadata.timestamp
    };
    
    await this.orderSummaryCollection.insertOne(orderSummary);
    
    // Update customer orders aggregation
    await this.customerOrdersCollection.updateOne(
      { _id: event.eventData.customerId },
      {
        $inc: { 
          totalOrders: 1,
          totalSpent: event.eventData.totalAmount
        },
        $set: { lastOrderDate: event.metadata.timestamp },
        $setOnInsert: { firstOrderDate: event.metadata.timestamp }
      },
      { upsert: true }
    );
  }
  
  async handleOrderStatusChanged(event) {
    await this.orderSummaryCollection.updateOne(
      { _id: event.eventData.orderId },
      {
        $set: {
          status: event.eventData.newStatus,
          lastModified: event.metadata.timestamp
        }
      }
    );
    
    // Update additional projections based on status
    if (event.eventData.newStatus === 'Completed') {
      await this.customerOrdersCollection.updateOne(
        { _id: event.eventData.customerId },
        { $inc: { completedOrders: 1 } }
      );
    }
  }
}

// Custom role definitions for enterprise security
const securityRoles = {
  // Application service roles
  userServiceRole: {
    role: "userServiceRole",
    privileges: [
      {
        resource: { db: "userdb", collection: "users" },
        actions: ["find", "insert", "update", "remove"]
      },
      {
        resource: { db: "userdb", collection: "user_sessions" },
        actions: ["find", "insert", "remove"]
      }
    ],
    roles: []
  },
  
  // Analytics read-only role
  analyticsRole: {
    role: "analyticsRole",
    privileges: [
      {
        resource: { db: "", collection: "" }, // All databases
        actions: ["find", "listCollections", "listIndexes"]
      }
    ],
    roles: []
  },
  
  // Audit role with special permissions
  auditRole: {
    role: "auditRole",
    privileges: [
      {
        resource: { cluster: true },
        actions: ["auditLogRotate", "viewRole", "viewUser"]
      },
      {
        resource: { db: "audit", collection: "" },
        actions: ["find", "insert", "listCollections"]
      }
    ],
    roles: []
  }
};

// User creation with role assignment
async function createApplicationUsers(adminDb) {
  // Create custom roles
  for (const [roleName, roleConfig] of Object.entries(securityRoles)) {
    try {
      await adminDb.runCommand({
        createRole: roleConfig.role,
        privileges: roleConfig.privileges,
        roles: roleConfig.roles
      });
      console.log(`Created role: ${roleConfig.role}`);
    } catch (error) {
      if (error.code !== 51002) { // Role already exists
        console.error(`Failed to create role ${roleConfig.role}:`, error);
      }
    }
  }
  
  // Create application users
  const applicationUsers = [
    {
      user: "userService",
      pwd: process.env.USER_SERVICE_PASSWORD,
      roles: ["userServiceRole"]
    },
    {
      user: "analyticsService",
      pwd: process.env.ANALYTICS_SERVICE_PASSWORD,
      roles: ["analyticsRole"]
    },
    {
      user: "auditService",
      pwd: process.env.AUDIT_SERVICE_PASSWORD,
      roles: ["auditRole"]
    }
  ];
  
  for (const userConfig of applicationUsers) {
    try {
      await adminDb.runCommand({
        createUser: userConfig.user,
        pwd: userConfig.pwd,
        roles: userConfig.roles
      });
      console.log(`Created user: ${userConfig.user}`);
    } catch (error) {
      if (error.code !== 51003) { // User already exists
        console.error(`Failed to create user ${userConfig.user}:`, error);
      }
    }
  }
}

// Client-side field level encryption setup
const { MongoClient } = require('mongodb');
const { ClientEncryption } = require('mongodb-client-encryption');

class FieldLevelEncryption {
  constructor(uri, keyVaultNamespace) {
    this.uri = uri;
    this.keyVaultNamespace = keyVaultNamespace;
    this.kmsProviders = {
      local: {
        key: Buffer.from(process.env.MASTER_KEY, 'base64')
      }
    };
  }
  
  async initialize() {
    // Create key vault client
    this.keyVaultClient = new MongoClient(this.uri);
    await this.keyVaultClient.connect();
    
    // Create client encryption
    this.clientEncryption = new ClientEncryption(
      this.keyVaultClient,
      {
        keyVaultNamespace: this.keyVaultNamespace,
        kmsProviders: this.kmsProviders
      }
    );
    
    // Create data encryption keys
    this.dataKeys = {
      ssn: await this.createDataKey('SSN_KEY'),
      creditCard: await this.createDataKey('CREDIT_CARD_KEY'),
      personalData: await this.createDataKey('PERSONAL_DATA_KEY')
    };
  }
  
  async createDataKey(altName) {
    try {
      return await this.clientEncryption.createDataKey('local', {
        keyAltNames: [altName]
      });
    } catch (error) {
      // Key might already exist, try to find it
      const keyVault = this.keyVaultClient
        .db(this.keyVaultNamespace.split('.')[0])
        .collection(this.keyVaultNamespace.split('.')[1]);
        
      const existingKey = await keyVault.findOne({
        keyAltNames: altName
      });
      
      return existingKey ? existingKey._id : null;
    }
  }
  
  async encryptField(value, fieldType, algorithm = 'AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic') {
    const keyId = this.dataKeys[fieldType];
    if (!keyId) {
      throw new Error(`No encryption key found for field type: ${fieldType}`);
    }
    
    return await this.clientEncryption.encrypt(value, {
      keyId,
      algorithm
    });
  }
  
  async decryptField(encryptedValue) {
    return await this.clientEncryption.decrypt(encryptedValue);
  }
  
  // Create encrypted client with automatic encryption
  async createEncryptedClient() {
    const schemaMap = {
      "healthcare.patients": {
        bsonType: "object",
        properties: {
          ssn: {
            encrypt: {
              keyId: this.dataKeys.ssn,
              bsonType: "string",
              algorithm: "AEAD_AES_256_CBC_HMAC_SHA_512-Deterministic"
            }
          },
          medicalRecord: {
            encrypt: {
              keyId: this.dataKeys.personalData,
              bsonType: "object",
              algorithm: "AEAD_AES_256_CBC_HMAC_SHA_512-Random"
            }
          }
        }
      }
    };
    
    const client = new MongoClient(this.uri, {
      autoEncryption: {
        keyVaultNamespace: this.keyVaultNamespace,
        kmsProviders: this.kmsProviders,
        schemaMap
      }
    });
    
    await client.connect();
    return client;
  }
}

// Advanced audit logging configuration
const auditConfiguration = {
  // Enable audit logging
  auditLog: {
    destination: "file",
    path: "/var/log/mongodb/audit.json",
    format: "JSON",
    filter: {
      // Audit all authentication events
      $or: [
        { "atype": "authenticate" },
        { "atype": "authCheck" },
        { "atype": "logout" },
        
        // Audit administrative operations
        { "atype": "createCollection" },
        { "atype": "dropCollection" },
        { "atype": "createIndex" },
        { "atype": "dropIndex" },
        
        // Audit data modification on sensitive collections
        {
          $and: [
            { "atype": { $in: ["insert", "update", "remove"] } },
            { "param.ns": { $regex: "^(users|payments|medical)" } }
          ]
        },
        
        // Audit role and user management
        { "atype": "createUser" },
        { "atype": "dropUser" },
        { "atype": "createRole" },
        { "atype": "dropRole" },
        { "atype": "grantRolesToUser" },
        { "atype": "revokeRolesFromUser" }
      ]
    }
  }
};

// Custom audit event handler
class AuditEventProcessor {
  constructor(db) {
    this.db = db;
    this.auditCollection = db.collection('security_audit_events');
  }
  
  async processAuditEvent(auditEvent) {
    const processedEvent = {
      ...auditEvent,
      processedAt: new Date(),
      riskScore: this.calculateRiskScore(auditEvent),
      category: this.categorizeEvent(auditEvent)
    };
    
    await this.auditCollection.insertOne(processedEvent);
    
    // Check for suspicious patterns
    if (processedEvent.riskScore > 7) {
      await this.triggerSecurityAlert(processedEvent);
    }
  }
  
  calculateRiskScore(event) {
    let score = 0;
    
    // Failed authentication attempts
    if (event.atype === 'authenticate' && event.result === 18) {
      score += 3;
    }
    
    // Administrative operations outside business hours
    const hour = new Date(event.ts).getHours();
    if (['createUser', 'dropUser', 'createRole', 'dropRole'].includes(event.atype) &&
        (hour < 6 || hour > 22)) {
      score += 5;
    }
    
    // Bulk data access
    if (event.atype === 'find' && event.param?.filter === {}) {
      score += 4;
    }
    
    // Unusual source IP
    if (event.remote && !this.isKnownIP(event.remote)) {
      score += 3;
    }
    
    return Math.min(score, 10);
  }
  
  categorizeEvent(event) {
    const categories = {
      'authenticate': 'AUTHENTICATION',
      'authCheck': 'AUTHORIZATION',
      'insert': 'DATA_MODIFICATION',
      'update': 'DATA_MODIFICATION',
      'remove': 'DATA_MODIFICATION',
      'find': 'DATA_ACCESS',
      'createUser': 'USER_MANAGEMENT',
      'dropUser': 'USER_MANAGEMENT',
      'createRole': 'ROLE_MANAGEMENT',
      'dropRole': 'ROLE_MANAGEMENT'
    };
    
    return categories[event.atype] || 'OTHER';
  }
  
  async triggerSecurityAlert(event) {
    const alert = {
      alertId: new ObjectId(),
      type: 'HIGH_RISK_AUDIT_EVENT',
      severity: 'HIGH',
      event: event,
      triggeredAt: new Date(),
      status: 'OPEN',
      assignedTo: null
    };
    
    await this.db.collection('security_alerts').insertOne(alert);
    
    // Send notification to security team
    await this.notifySecurityTeam(alert);
  }
  
  async notifySecurityTeam(alert) {
    // Implementation for security team notification
    console.log(`🚨 Security Alert: ${alert.type} - Risk Score: ${alert.event.riskScore}`);
  }
  
  isKnownIP(ip) {
    const knownIPRanges = [
      '10.0.0.0/8',
      '172.16.0.0/12',
      '192.168.0.0/16'
    ];
    
    // Implementation for IP range checking
    return true; // Simplified for example
  }
}

// Comprehensive performance monitoring system
class MongoDBPerformanceMonitor {
  constructor(db, alertManager) {
    this.db = db;
    this.alertManager = alertManager;
    this.metricsCollection = db.collection('performance_metrics');
    this.alertCollection = db.collection('performance_alerts');
  }
  
  async collectPerformanceMetrics() {
    const metrics = {
      timestamp: new Date(),
      server: await this.getServerStatus(),
      database: await this.getDatabaseMetrics(),
      queries: await this.getSlowQueryMetrics(),
      replication: await this.getReplicationMetrics(),
      sharding: await this.getShardingMetrics()
    };
    
    await this.metricsCollection.insertOne(metrics);
    await this.analyzeMetrics(metrics);
    
    return metrics;
  }
  
  async getServerStatus() {
    const serverStatus = await this.db.admin().serverStatus();
    
    return {
      uptime: serverStatus.uptime,
      memory: {
        resident: serverStatus.mem.resident,
        virtual: serverStatus.mem.virtual,
        mapped: serverStatus.mem.mapped
      },
      connections: {
        current: serverStatus.connections.current,
        available: serverStatus.connections.available,
        totalCreated: serverStatus.connections.totalCreated
      },
      network: {
        bytesIn: serverStatus.network.bytesIn,
        bytesOut: serverStatus.network.bytesOut,
        numRequests: serverStatus.network.numRequests
      },
      opcounters: serverStatus.opcounters,
      wiredTiger: serverStatus.wiredTiger ? {
        cacheSize: serverStatus.wiredTiger.cache['bytes currently in the cache'],
        cacheDirtySize: serverStatus.wiredTiger.cache['tracked dirty bytes in the cache'],
        cacheReadIntoSize: serverStatus.wiredTiger.cache['bytes read into cache'],
        cacheWrittenFromSize: serverStatus.wiredTiger.cache['bytes written from cache']
      } : null
    };
  }
  
  async getDatabaseMetrics() {
    const dbStats = await this.db.stats();
    
    return {
      collections: dbStats.collections,
      objects: dbStats.objects,
      avgObjSize: dbStats.avgObjSize,
      dataSize: dbStats.dataSize,
      storageSize: dbStats.storageSize,
      indexSize: dbStats.indexSize,
      indexes: dbStats.indexes
    };
  }
  
  async getSlowQueryMetrics() {
    // Enable profiling for slow operations
    await this.db.runCommand({ profile: 2, slowms: 100 });
    
    const slowQueries = await this.db.collection('system.profile')
      .find({ ts: { $gte: new Date(Date.now() - 5 * 60 * 1000) } }) // Last 5 minutes
      .sort({ ts: -1 })
      .limit(100)
      .toArray();
    
    return {
      count: slowQueries.length,
      avgDuration: slowQueries.reduce((sum, q) => sum + q.millis, 0) / slowQueries.length || 0,
      queries: slowQueries.slice(0, 10) // Top 10 slowest
    };
  }
  
  async getReplicationMetrics() {
    try {
      const replStatus = await this.db.admin().replSetGetStatus();
      
      return {
        state: replStatus.myState,
        members: replStatus.members.map(member => ({
          name: member.name,
          state: member.state,
          health: member.health,
          uptime: member.uptime,
          optime: member.optime,
          syncSourceHost: member.syncSourceHost
        })),
        replicationLag: this.calculateReplicationLag(replStatus.members)
      };
    } catch (error) {
      return { error: 'Not in replica set or insufficient permissions' };
    }
  }
  
  async getShardingMetrics() {
    try {
      const shardingStatus = await this.db.admin().runCommand({ listShards: 1 });
      const configDB = this.db.getSiblingDB('config');
      
      const chunks = await configDB.chunks.countDocuments();
      const balancerState = await configDB.settings.findOne({ _id: 'balancer' });
      
      return {
        shards: shardingStatus.shards.length,
        chunks: chunks,
        balancerEnabled: balancerState ? balancerState.stopped !== true : true
      };
    } catch (error) {
      return { error: 'Not in sharded cluster or insufficient permissions' };
    }
  }
  
  calculateReplicationLag(members) {
    const primary = members.find(m => m.state === 1);
    if (!primary) return null;
    
    const secondaries = members.filter(m => m.state === 2);
    const maxLag = Math.max(...secondaries.map(s => 
      primary.optime.ts.getTime() - s.optime.ts.getTime()
    ));
    
    return maxLag / 1000; // Convert to seconds
  }
  
  async analyzeMetrics(metrics) {
    const alerts = [];
    
    // Connection alert
    if (metrics.server.connections.current / metrics.server.connections.available > 0.8) {
      alerts.push({
        type: 'HIGH_CONNECTION_USAGE',
        severity: 'WARNING',
        message: `Connection usage at ${Math.round(metrics.server.connections.current / metrics.server.connections.available * 100)}%`,
        metrics: {
          current: metrics.server.connections.current,
          available: metrics.server.connections.available
        }
      });
    }
    
    // Memory alert
    if (metrics.server.memory.resident > 8000) { // 8GB threshold
      alerts.push({
        type: 'HIGH_MEMORY_USAGE',
        severity: 'WARNING',
        message: `Resident memory usage: ${metrics.server.memory.resident}MB`,
        metrics: { resident: metrics.server.memory.resident }
      });
    }
    
    // Slow query alert
    if (metrics.queries.count > 50) {
      alerts.push({
        type: 'HIGH_SLOW_QUERY_RATE',
        severity: 'CRITICAL',
        message: `${metrics.queries.count} slow queries in last 5 minutes`,
        metrics: {
          count: metrics.queries.count,
          avgDuration: metrics.queries.avgDuration
        }
      });
    }
    
    // Replication lag alert
    if (metrics.replication.replicationLag > 10) {
      alerts.push({
        type: 'HIGH_REPLICATION_LAG',
        severity: 'CRITICAL',
        message: `Replication lag: ${metrics.replication.replicationLag} seconds`,
        metrics: { lag: metrics.replication.replicationLag }
      });
    }
    
    // Process and store alerts
    for (const alert of alerts) {
      await this.processAlert(alert, metrics.timestamp);
    }
  }
  
  async processAlert(alert, timestamp) {
    const alertDocument = {
      ...alert,
      timestamp,
      status: 'ACTIVE',
      acknowledged: false,
      resolvedAt: null
    };
    
    await this.alertCollection.insertOne(alertDocument);
    await this.alertManager.sendAlert(alertDocument);
  }
}

// Automated backup and disaster recovery
class BackupManager {
  constructor(mongoUri, s3Config) {
    this.mongoUri = mongoUri;
    this.s3Config = s3Config;
  }
  
  async createBackup(databases = []) {
    const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
    const backupId = `backup-${timestamp}`;
    
    try {
      for (const dbName of databases) {
        await this.backupDatabase(dbName, backupId);
      }
      
      await this.uploadToS3(backupId);
      await this.recordBackup(backupId, databases);
      
      return backupId;
    } catch (error) {
      console.error(`Backup failed for ${backupId}:`, error);
      throw error;
    }
  }
  
  async backupDatabase(dbName, backupId) {
    const backupPath = `/tmp/backups/${backupId}/${dbName}`;
    
    // Use mongodump for consistent backup
    const { execSync } = require('child_process');
    const command = `mongodump --uri="${this.mongoUri}" --db=${dbName} --out=${backupPath}`;
    
    execSync(command, { stdio: 'inherit' });
  }
  
  async uploadToS3(backupId) {
    // Implementation for S3 upload
    console.log(`Uploading backup ${backupId} to S3...`);
  }
  
  async recordBackup(backupId, databases) {
    const client = new MongoClient(this.mongoUri);
    await client.connect();
    
    const adminDb = client.db('admin');
    await adminDb.collection('backups').insertOne({
      backupId,
      databases,
      createdAt: new Date(),
      status: 'COMPLETED',
      location: `s3://${this.s3Config.bucket}/backups/${backupId}`
    });
    
    await client.close();
  }
}