I've integrated BullMQ into our Node.js application, leveraging Redis Cluster for production. According to the BullMQ and Redis Cluster documentation I understood that we can give some prefix in key while creating queue in bullMQ which internally takes CRC16 of the key modulo 16384 (https://docs.bullmq.io/bull/patterns/redis-cluster) but this doesn't ensures that each queue will push data to different node and can result in hot shard
Is there a recommended way or strategy to ensure that each queue in BullMQ pushes data to different Redis nodes within the cluster, thereby avoiding potential performance bottlenecks caused by uneven data distribution? Any insights or best practices would be greatly appreciated.
Below is my demo code:
const Redis = require('ioredis');const { Queue, Worker } = require('bullmq');const clusterNodes = [ { port: 6380, host: "127.0.0.1" }, { port: 6381, host: "127.0.0.1" }, { port: 6382, host: "127.0.0.1" }];const cluster = new Redis.Cluster(clusterNodes);async function bullMqTest() { const queue1 = new Queue(`myQueue1`, { connection: cluster, prefix: "{queue_1}", }); const queue2 = new Queue(`myQueue2`, { connection: cluster, prefix: "{queue_2}", }); const queue3 = new Queue(`myQueue3`, { connection: cluster, prefix: "{queue_3}", }); const queue4 = new Queue(`myQueue4`, { connection: cluster, prefix: "{queue_4}", }); const queue5 = new Queue(`myQueue5`, { connection: cluster, prefix: "{queue_5}", }); const queue6 = new Queue(`myQueue6`, { connection: cluster, prefix: "{queue_6}", }); queue1.add('myJob', { foo: 'bar' }); queue2.add('myJob', { foo: 'baz' }); queue3.add('myJob', { foo: 'qux' }); queue4.add('myJob', { foo: 'xyz' }); queue5.add('myJob', { foo: 'lmn' }); queue6.add('myJob', { foo: 'qpr' });}module.exports = { pushDataToRedis, fetchDataFromRedis, bullMqTest };