Implementation of data warehousing by asynchronous Redis queues

Business requirements

The app client sends the JSON data to the server interface once a day, emptying the cache and sending it again.

Business logic before a problem:

The PHP interface first converts JSON to an array to insert nonexistent data in a large table

The user already exists and the new ID

into a different table of particulars

The problem lies in:

When the user clears the cache due to special circumstances, the app sends the JSON string into the warehouse, causing the CPU to burst to 88% and remain high.

Optimization ideas:

1. Asynchronous Queue Processing

2, Redis filter (is only to deal with the first request of the day)

3. Redis Secondary Storage app name (after verification, bulk insert data in App Name table)

4, splicing insert and add as detailed table

Solution:

1, Interface modification Redis filter + such as List queue and deposit the result into Redis

First, Redis the previous historical data in the Redis hash. The Key name ID is the key value

<?php/** * Created by Haiyong.


* User:jia * DATE:2017/9/18 * time:20:06 * * namespace App\http\controllers\app;
Use App\http\controllers\controller;
Use Illuminate\http\request;
Use illuminate\support\facades\db;

Use Illuminate\support\facades\redis; Class Otherappcontroller extends controller{/** * App Application Statistics interface * @param Request $request * @return Strin G/Public Function apptotal (Request $request) {////Historical data warehousing//$redis = Redis::connection ('
        Web_active '); $app _name = db::connection (' Phplog ')->table (' App_set_name ')->where ("AppName", ' <> ', ')->lists ('
        Id ', ' appName ');
        $str = ';
        foreach ($app _name as $key => $val) {//$str. = "{$val} {$key}";
        }//$redis->hmset (' app_name ', $app _name);
        Echo $str, exit;
        $result = $request->input (' res ');
        $list = Json_decode ($result, true); if (Empty ($list) | |!is_array ($lIST)) {return Json_encode ([' Result ' => ' error ', ' msg ' => ' parameter error ')]; $data [' uid '] = isset ($list [' uid '])?
        $list [' uid ']: ' 20001 ';
        $data [' time '] = date (' y-m-d ');
        $redis _key = ' Log_app: '. $data [' time '];
        Redis Filter $redis = redis::connection (' web_active ');
            Redis key value expiration setting if (Empty ($redis->exists ($redis _key)) {$redis->hset ($redis _key, 1, ' Start '); $redis->expireat ($redis _key, Strtotime ($data [' time '].
        +2 Day '));  The//value determines if ($redis->hexists ($redis _key, $data [' uid ')]) {return Json_encode ([' Result ' =>
        ' SUCCESS ']);
            else {//push queue $redis->hset ($redis _key, $data [' uid '], $result); $redis->rpush (' log_app_list ', $data [' time ']. ':' .
            $data [' uid ']);
        return Json_encode ([' Result ' => ' SUCCESS ']);


 }
    }
}

2, PHP script loop monitoring Redis queue execution logic to prevent memory overflow

Mget returns null if the user's app ID does not exist

By judging null using the Redis new value as a self-increasing ID pointer to fill in MySQL and following the new Redis hash and pointer values and warehousing details table

<?php namespace App\console\commands;
Use Illuminate\console\command;
Use Illuminate\support\facades\redis;
Use illuminate\support\facades\db;

Use Illuminate\support\facades\storage;
     Class Apptotal extends Command {/** * The name and signature of the console command.

    * * @var string/protected $signature = ' apptotal:run ';
     /** * the console command description.

    * * @var String */protected $description = ' Command description ';
     /** * Create A new command instance.
    * * @return void */Public function __construct () {parent::__construct ();
     }/** * Execute the console command. * * @return Mixed */public function handle () {//Historical data warehousing//$redis = Redis::connectio  
        N (' web_active '); $app _name = db::connection (' Phplog ')->table (' App_set_name ')->where ("AppName", ' <> ', ')->lists ('  
        Id ', ' appName '); // $Redis->hmset (' app_name ', $app _name);  
          Exit
            while (1) {$redis = redis::connection (' web_active ');   
            Queue name $res = $redis->lpop (' log_app_list ');
            Switch button $lock = $redis->get (' Log_app_lock ');
                if (!empty ($res)) {list ($date, $uid) = Explode (': ', $res);
                $result = $redis->hget (' Log_app: '. $date, $uid);
                        if (!empty ($result)) {$table _name = ' app_total '. Date (' Ym ');
                        $list = Json_decode ($result, true); $data [' uid '] = isset ($list [' uid '])?
                        $list [' uid ']: ' 20001 '; $data [' sex '] = isset ($list [' sex '])?
                        $list [' Sex ']: '; $data [' device '] = isset ($list [' Device '])?
                        $list [' Device ']: '; $data [' applist '] = isset ($list [' list '])?
                        $list [' list ']: ';
          Data to flip is more performance-saving than unique              $data [' applist '] = Array_flip ($data [' applist ']);
                        $data [' applist '] = Array_flip ($data [' applist ']);
                        $data [' time '] = date (' y-m-d ');
                        App Application filter $app _res = $redis->hmget (' app_name ', $data [' applist ']);
                        New Add app array $new _app = [];
				        MySQL storage array $mysql _new_app = [];
				        Gets the current Redis pointer $total = $redis->get (' app_name_total ');
				                foreach ($app _res as $key =>& $val) {if (Is_null ($val)) {$total = 1; 
				                $new _app[$data [' applist '] [$key]] = $total;
				                $val = $total;
				            Array_push ($mysql _new_app,[' id ' => $total, ' appName ' => $data [' applist '] [$key]]); } if (count ($new _app)) {$str = INSERT IGNORE into app_set_name(id,appname) values ";
				        	foreach ($new _app as $key => $val) {$str. = "(. $val.", ' ". $key." '), ";
				        	$str = Trim ($str, ', ');
				        	$mysql _res = db::connection (' Phplog ')->table (' App_set_name ')->insert ($mysql _new_app);
						        $mysql _res = db::connection (' Phplog ')->statement ($STR);
						            if ($mysql _res) {//Set Redis pointer $redis->set (' app_name_total ', $total);
						        Redis Data Warehousing $redis->hmset (' app_name ', $new _app);
                        }//Details Data warehousing $data [' applist '] = implode (', ', $app _res); App stats warehousing db::connection (' Phplog ')->statement ("INSERT IGNORE into". $table _name. " (Uid,sex,device, ' time ', applist) VALUES (' ". $data [' uid ']." ', '. $data [' sex ']. $data [' Device ']. $data ['
                        Time ']. "', '". $data [' applist ']. "]);  Log log Generate memory error when file reaches 123MB all this place is using log cutting or not writing log Storage::d ISK (' local ')->append (Directory_separat OR. ' Total '.  Directory_separator ' LoaAppTotal.txt ', date (' y-m-d h:i:s '). ' Success '. $result. " 
				    \ n "); else {Storage::d ISK (' local ')->append (directory_separator. ' Total ').  Directory_separator ' LoaAppTotal.txt ', date (' y-m-d h:i:s '). ' Error '. $result. "
				    \ n ");
            }///Perform interval sleep (1);
            The end button if ($lock = = 2) {exit; }//Memory Detect if (Memory_get_usage () >1000*1024*1024) {exit (' memory overflow ');//greater than 100M memory exit program to prevent internal
 Save leak is killed by System cause task Terminal}}}

3, fixed Timing task monitoring script implementation

Crontab-e

/2 * * * * */bin/bash/usr/local/nginx/html/test.sh 1>>/usr/local/nginx/html/log.log 2>&1

test.sh content (view the process ID returned by the execution command if the command is not opened)

#!/bin/bash
Alive= ' Ps-ef | grep apptotal | Grep-v grep | awk ' {print $} '
if [! $alive]
Then
/usr/local/php/bin/php/var/ms/artisan apptotal:run >/dev/null &
Fi

Remember to authorize OH chmod +x test.sh

The author uses the Laravel framework to throw command activation into the background

Execute command

/usr/local/php/bin/php/var/ms/artisan apptotal:run >/dev/null &

It's done. Direct ctrl-c end on line command to run in the background you can view the process ID with commands in the shell

This enables the asynchronous storage of queues

There are many problems to be optimized. The general function has been implemented ...

Optimized CPU after completion