Q-learning Concise example Java code implementation

In the concise example of q-learning, we introduce the simple examples of q-learning algorithm, from which we can summarize the basic idea of q-learning algorithm.

Experience score for this selection = Feedback score for this selection + historical best experience score for the scene after this selection

The feedback score is the value score (fixed score) of a single step, and the experience score is the learning score (dynamic score) of the completion goal.

The Java implementation of a concise example is as follows

 Packagecom.coshaho.learn.qlearning;Importjava.util.ArrayList;Importjava.util.Collections;Importjava.util.List;ImportJava.util.Random;/*** * Qlearning.java Create on September 4, 2017 10:08:49 * class function Description: Qlearning Concise Example Implementation * * Copyright:copyright (c) 2 013 * Company:coshaho * @Version 1.0 * @Author Coshaho*/ Public classqlearning {Feedbackmatrix R=NewFeedbackmatrix (); Experiencematrix Q=NewExperiencematrix ();  Public Static voidMain (string[] args) {qlearning QL=Newqlearning ();  for(inti = 0; I < 500; i++) {Random random=NewRandom (); intx = random.nextint (100)% 6; System.out.println ("First" + i + "secondary study, initial room is" +x);            Ql.learn (x);        System.out.println (); }    }         Public voidLearn (intx) { Do        {            //randomly select a unicom room to enter            inty =Chooserandomry (x); //get the historical best score for the starting point of the room to enter            intQY =getmaxqy (y); //calculate the score for this move            intValue =calculatenewq (x, y, qy);            Q.set (x, y, value); X=y; }        //go out of the room and learn to end         while(5! =x);    Q.print (); }         Public intChooserandomry (intx) {int[] Qrow =R.getrow (x); List<Integer> yvalues =NewArraylist<integer>();  for(inti = 0; i < qrow.length; i++)        {            if(Qrow[i] >= 0) {yvalues.add (i); }} random Random=NewRandom (); inti = Random.nextint (yvalues.size ())%yvalues.size (); returnYvalues.get (i); }         Public intGETMAXQY (intx) {int[] Qrow =Q.getrow (x); intLength =qrow.length; List<YAndValue> yvalues =NewArraylist<yandvalue>();  for(inti = 0; i < length; i++) {Yandvalue YV=NewYandvalue (i, qrow[i]);        Yvalues.add (YV);        } collections.sort (Yvalues); intnum = 1; intValue = Yvalues.get (0). GetValue ();  for(inti = 1; i < length; i++)        {            if(Yvalues.get (i). GetValue () = =value) {num= i + 1; }            Else            {                 Break; }} random Random=NewRandom (); inti = random.nextint (num)%num; returnYvalues.get (i). GetY (); }        //q (x, y) = R (x, y) + 0.8 * MAX (Q (y,i))     Public intCALCULATENEWQ (intXintYintqy) {        return(int) (R.get (x, y) + 0.8 *q.get (y, qy)); }         Public Static classYandvalueImplementsComparable<yandvalue>    {        inty; intvalue;  Public intGetY () {returny; }         Public voidSety (inty) { This. y =y; }         Public intGetValue () {returnvalue; }         Public voidSetValue (intvalue) {             This. Value =value; }         PublicYandvalue (intYintvalue) {             This. y =y;  This. Value =value; }         Public intcompareTo (Yandvalue o) {returnO.getvalue ()- This. Value; }    }} Packagecom.coshaho.learn.qlearning;/*** * Feedbackmatrix.java Create on September 4, 2017 9:52:41 * class function Description: Feedback Matrix * * Copyright:copyright (c) 2013 * Company:coshaho * @Version 1.0 * @Author Coshaho*/ Public classFeedbackmatrix { Public intGetintXinty) {returnR[x][y]; }         Public int[] GetRow (intx) {returnR[x]; }        Private Static int[] R =New int[6] [6]; Static{r[0][0] = 1; r[0][1] = 1; r[0][2] = 1; r[0][3] = 1; r[0][4] = 0; r[0][5] = 1; r[1][0] = 1; r[1][1] = 1; r[[1]; r[1][3] = 0; r[1][4] = 1; r[1][5] = 100; r[2][0] = 1; r[2][1] = 1; r[2][2] = 1; r[2][3] = 0; r[2][4] = 1; r[2][5] = 1; r[3][0] = 1; r[3][1] = 0; r[3][2] = 0; r[3][3] = 1; r[3][4] = 0; r[3][5] = 1; r[4][0] = 0; r[4][1] = 1; r[4][2] = 1; r[4][3] = 0; r[4][4] = 1; r[4][5] = 100; r[5][0] = 1; r[5][1] = 0; r[5][2] = 1; r[5][3] = 1; r[5][4] = 0; r[5][5] = 100; }} Packagecom.coshaho.learn.qlearning;/*** * Experiencematrix.java Create on September 4, 2017 10:03:08 * class function Description: Experience Matrix * * Copyright:copyright (c) 2013 * Company:coshaho * @Version 1.0 * @Author Coshaho*/ Public classExperiencematrix { Public intGetintXinty) {returnQ[x][y]; }         Public int[] GetRow (intx) {returnQ[x]; }         Public voidSetintXintYintvalue) {Q[x][y]=value; }         Public voidprint () { for(inti = 0; I < 6; i++)        {             for(intj = 0; J < 6; J + +) {String s= Q[i][j] + ""; if(Q[i][j] < 10) {s= S + ""; }                Else if(Q[i][j] < 100) {s= S + "";            } System.out.print (s);        } System.out.println (); }    }        Private Static int[] Q =New int[6] [6]; Static{q[0][0] = 0; q[0][1] = 0; q[0][2] = 0; q[0][3] = 0; q[0][4] = 0; q[0][5] = 0; q[1][0] = 0; q[1][1] = 0; q[[0]; q[1][3] = 0; q[1][4] = 0; q[1][5] = 0; q[2][0] = 0; q[2][1] = 0; q[2][2] = 0; q[2][3] = 0; q[2][4] = 0; q[2][5] = 0; q[3][0] = 0; q[3][1] = 0; q[3][2] = 0; q[3][3] = 0; q[3][4] = 0; q[3][5] = 0; q[4][0] = 0; q[4][1] = 0; q[4][2] = 0; q[4][3] = 0; q[4][4] = 0; q[4][5] = 0; q[5][0] = 0; q[5][1] = 0; q[5][2] = 0; q[5][3] = 0; q[5][4] = 0; q[5][5] = 0; }}

After 500 calculations, the following results are obtained

No. 499 time Study, initial room is    0    0    0    396  0    0    0 0  0    496  0    0    0  0    0    0    396  252  0    396 0 A  0    0    316  0    496  0    396  0    0    396  496  

At this point, we enter from any room, each time we select the highest score step to move, you can always find the shortest escape path.

Q-learning Concise example Java code implementation