Node. js + Redis Sorted Set to implement task queue, node. jsredis

Node. js + Redis Sorted Set to implement task queue, node. jsredis

Requirement: function A needs to call A third-party API to obtain data, while the third-party API itself is an asynchronous processing method. After the call, data and status {data: "query result", "status ": "In asynchronous processing"}, so you need to call a third-party API to obtain data after a period of time. In order that you do not have to wait because the third-party API is being processed asynchronously when using function A, add the user request to the task queue, return some data, and close the request. Then, the task is periodically retrieved from the task queue to call a third-party API. If the returned status is "Asynchronous processing", the task is added to the task queue again, if the returned status is "processed", the returned data is stored in the database.

Based on the above questions, I want to use Node. js + Redis sorted set to implement task queue. Node. js implements its own application API to accept user requests, merge the existing database data and some data returned by the API to the user, and add the task to the task queue. Node. js child process and cron are used to regularly retrieve and execute tasks from the task queue.

Several issues that need to be considered in designing the task queue

• Execute multiple tasks in parallel
• Task uniqueness
• Processing after the task is successful or failed

Solutions to the above problems

• Execute multiple tasks in parallel using Promise. all.
• Task uniqueness is achieved using Redis sorted set. You can use a timestamp as a score to use sorted set as a list. When you add a task, you can determine whether the task exists. When the task is executed, you can set the score to 0, each time a task with a score greater than 0 is taken for execution, it can avoid repeated execution of the task.
• After the task is successfully executed, delete the task. After the task fails, update the task score to the current timestamp. In this way, you can add the failed task to the end of the task queue.

Sample Code

// Remote_api.js simulates a third-party API 'use strict '; const app = require ('express') (); app. get ('/', (req, res) =>{ setTimeout () =>{ let arr = [200,300]; // 200 indicates success, 300 indicates that a new request to res is required for failure. status (200 ). send ({'status': arr [parseInt (Math. random () * 2)]}) ;}, 3000) ;}); app. listen ('20140901', () => {console. log ('api listening port: 100') ;}); // producer. js uses APIs to accept user requests and add tasks to the task queue 'use strict '. const app = require ('express') (); c Onst redisClient = require ('redis '). createClient (); const QUEUE_NAME = 'queue: example '; function addTaskToQueue (taskName, callback) {// first, judge whether the task already exists, exist: Skip, does not exist: add the redisClient to the task queue. zscore (QUEUE_NAME, taskName, (error, task) =>{ if (error) {console. log (error);} else {if (task) {console. log ('Task already exists, do not add the same task'); callback (null, task);} else {redisClient. zadd (QUEUE_NAME, new Date (). getTime (), taskName, (Error, result) =>{ if (error) {callback (error) ;}else {callback (null, result) ;}}});} app. get ('/', (req, res) => {let taskName = req. query ['Task-name']; addTaskToQueue (taskName, (error, result) =>{ if (error) {console. log (error);} else {res. status (200 ). send ('querying ...... ') ;}}) ;}); app. listen (9002, () => {console. log ('producer service listening port: 100') ;}); // consumer. js regularly retrieves tasks and executes 'use strict '; const RedisClient = require ('redis '). createClient (); const request = require ('request'); const schedule = require ('node-schedule'); const QUEUE_NAME = 'queue: expmple'; const PARALLEL_TASK_NUMBER = 2; // Number of concurrently executed tasks function getTasksFromQueue (callback) {// obtain multiple tasks redisClient. zrangebyscore ([QUEUE_NAME, 1, new Date (). getTime (), 'limit ', 0, PARALLEL_TASK_NUMBER], (error, tasks) =>{ if (error) {callback (error );} Else {// set the task score to 0, indicating that if (tasks. length> 0) {let tmp = []; tasks. forEach (task) => {tmp. push (0); tmp. push (task) ;}); redisClient. zadd ([QUEUE_NAME]. concat (tmp), (error, result) =>{ if (error) {callback (error);} else {callback (null, tasks) }}) ;}}});} function addFailedTaskToQueue (taskName, callback) {redisClient. zadd (QUEUE_NAME, new Date (). getTime (), taskName, (error, result) => {I F (error) {callback (error) ;}else {callback (null, result) ;}}) ;}function removeSucceedTaskFromQueue (taskName, callback) {redisClient. zrem (QUEUE_NAME, taskName, (error, result) =>{ if (error) {callback (error) ;}else {callback (null, result );}})} function execTask (taskName) {return new Promise (resolve, reject) =>{ let requestOptions = {'url': 'http: // 127.0.0.1: 9001 ', 'method ': 'get','t Imeout ': 5000}; request (requestOptions, (error, response, body) =>{ if (error) {resolve ('failed'); console. log (error); addFailedTaskToQueue (taskName, (error) =>{ if (error) {console. log (error) ;}else {}}) ;}else {try {body = typeof body! = 'Object '? JSON. parse (body): body;} catch (error) {resolve ('failed'); console. log (error); addFailedTaskToQueue (taskName, (error, result) =>{ if (error) {console. log (error) ;}else {}}); return ;}if (body. status! = 200) {resolve ('failed'); addFailedTaskToQueue (taskName, (error, result) =>{ if (error) {console. log (error) ;}else {}}) ;}else {resolve ('sudoed'); removeSucceedTaskFromQueue (taskName, (error, result) =>{ if (error) {console. log (error) ;}else {}}) ;}}}) ;}; // periodically, obtain a new task every five seconds to execute let job = schedule. scheduleJob ('*/5 ******', () => {console. log ('get new task'); getTasksFromQueue (error, tasks) =>{ if (error) {console. log (error);} else {if (tasks. length> 0) {console. log (tasks); Promise. all (tasks. map (execTask )). then (results) => {console. log (results );}). catch (error) => {console. log (error );});}}});});