Node.js + Redis Sorted set implements task queues _node.js

Requirements: function A needs to call a third-party API to get the data, and the third party API itself is asynchronous processing, after the call will return the data and state {information: "Query Results", "status": "In the Asynchronous Processing"}, so it will take a period of time before invoking the Third-party API to obtain Data. In order for a user to use feature A without waiting for a third-party API to be processed asynchronously, a user request is added to the task queue, a partial data is returned, and the request is closed. The task is then periodically removed from the task queue to call the Third-party API, if the return status is "asynchronous processing", the task is added to the task queue again, if the return status is "processed", will return the data warehousing.

Based on the above problems, think of using Node.js + Redis sorted set to implement task queues. Node.js implements its own application API to accept user requests, merge the stored data of the database with the data returned to the user by the API, and join the task in the task queue. Take the task execution from the task queue using the Node.js child process and cron timing.

Several issues to consider in the process of designing a task queue

• Perform multiple tasks in parallel
• Task uniqueness
• Processing after a task succeeds or fails

Solutions for the above problems

• Execute multiple tasks in parallel using Promise.all to implement
• Task uniqueness is implemented using Redis sorted set. Using the timestamp as a score can be implemented to use the sorted set as a list, when the task is added to determine whether the task already exists, when the task is taken out of the execution of the task is set to 0, every time the task to take a score greater than 0 to perform, you can avoid repetitive tasks.
• Delete the task after the task succeeds, and update the task score to the current timestamp when the task fails, so that the failed task can be rejoin the end of the task queue

Sample code

Remote_api.js Analog third party API ' use strict ';

Const APP = require (' Express ') (); App.get ('/', (req, res) => {settimeout () => {Let arr = [200, 300];//200 stands for Success, 300 for failure needs to be requested again Res.statu
  S. Send ({' Status ': Arr[parseint (Math.random () * 2)]});
}, 3000);

});
App.listen (' 9001 ', () => {console.log (' API Service listening port: 9001 ');});

Producer.js its own application API, which accepts user requests and joins the task queue ' use strict ';
Const APP = require (' Express ') ();

Const Redisclient = require (' Redis '). CreateClient ();

Const QUEUE_NAME = ' queue:example '; function Addtasktoqueue (TaskName, callback) {//First determine if the task already exists, exists: skipped, does not exist: Join task Queue Redisclient.zscore (queue_name, TaskName
    , (Error, Task) => {if (error) {Console.log (error);
        else {if (Task) {Console.log (' tasks already exist, do not add same task ');
      Callback (null, Task);  else {Redisclient.zadd (queue_name, New Date (). GetTime (), TaskName, (error, result) => {if (error)
          {Callback (Error);
     } else {       Callback (null, result);
      }
        });
}
    }
  });
  } app.get ('/', (req, res) => {Let taskname = req.query[' task-name '];
    Addtasktoqueue (TaskName, (error, result) => {if (error) {Console.log (error);
    else {res.status. Send (' in query ... ');;
}
  });

});
App.listen (9002, () => {console.log (' producer Service listening port: 9002 ');});

Consumer.js to get the task and execute the ' use strict ';
Const Redisclient = require (' Redis '). CreateClient ();
Const REQUEST = require (' request ');

Const SCHEDULE = require (' Node-schedule ');
Const QUEUE_NAME = ' queue:expmple '; Const PARALLEL_TASK_NUMBER = 2; Parallel execution Task Quantity function Gettasksfromqueue (callback) {//Get multiple task Redisclient.zrangebyscore ([Queue_name, 1, New Date (). Getting
    Time (), ' LIMIT ', 0, Parallel_task_number], (Error, tasks) => {if (error) {callback (error);
        else {//Set the task score to 0 to indicate that the if (Tasks.length > 0) {Let TMP = [] is being processed; Tasks.foreach (Task) => {TMP.PUSH (0);
        Tmp.push (Task);
        });
          Redisclient.zadd ([Queue_name].concat (TMP), (error, result) => {if (error) {callback (error);
      else {callback (null, Tasks)}});
}
    }
  }); function Addfailedtasktoqueue (TaskName, callback) {Redisclient.zadd (queue_name, New Date (). GetTime (), TaskName, (err
    Or, result) => {if (error) {callback (error);
    else {callback (null, result);
}
  }); function Removesucceedtaskfromqueue (TaskName, callback) {Redisclient.zrem (queue_name, TaskName, (error, result) =&gt ;
    {if (error) {callback (error);
    else {callback (null, result); }}} function Exectask (taskname) {return to New Promise (Resolve, reject) => {Let requestoptions = {' U
    RL ': ' http://127.0.0.1:9001 ', ' method ': ' Get ', ' Timeout ': 5000}; Request (Requestoptions, (Error, response, body) => {if (error){Resolve (' failed ');
        Console.log (Error);
          Addfailedtasktoqueue (TaskName, (Error) => {if (error) {Console.log (error);
      } else {}}); else {try {body = typeof body!== ' object '?
        Json.parse (body): the body;
          catch (Error) {Resolve (' failed ');
          Console.log (Error);
            Addfailedtasktoqueue (TaskName, (error, result) => {if (error) {Console.log (error);
          } else {}});
        Return
          } if (Body.status!==) {Resolve (' failed ');
            Addfailedtasktoqueue (TaskName, (error, result) => {if (error) {Console.log (error);
        } else {}});
          else {resolve (' succeed '); Removesucceedtaskfromqueue (TaskName, (error, result) => {if (error) {CONSOLE.LOG (Error);
        } else {}});
  }
      }
    });
});
  //timed, get new task every 5 seconds to execute let job = Schedule.schedulejob (' */5 * * * * * * * *, () => {console.log (' Get new Task ');
    Gettasksfromqueue (Error, tasks) => {if (error) {Console.log (error);

        else {if (Tasks.length > 0) {console.log (tasks);
        Promise.all (Tasks.map (Exectask)). Then (results) => {console.log (results);
        ). catch ((Error) => {console.log (error);
        
      });
}
    }
  });
 });