MapReduce implements the recommendation system

1. Development Environment:

Windows 2008 64bit

Java 1.6.0_30

Myhttp: //www.aliyun.com/zixun/aggregation/13428.html "> Eclipse 6.5

Environmental deployment see: http://www.linuxidc.com/Linux/2014-02/96528.htm

2.Hadoop cluster environment:

Oracle Linux Enterprise 5.9

Java 1.6.0_18

Hadoop: hadoop 1.2.1 Three nodes

namenode: 10.1.32.91

datanode: 10.1.32.93

datanode: 10.1.32.95

Environment Department see: http://www.linuxidc.com/Linux/2014-02/96529.htm

3 each java class features introduced:

Recommend.java, the main task to start the program

Step1.java, grouped by user, calculate the combined list of all items, and get the user's score matrix

Step2.java, the list of items to count the number of items to create the co-occurrence matrix

Step3.java, transformation of co-occurrence matrix and scoring matrix

Step4.java, merge the matrix, and calculate the list of recommended results

HdfsDAO.java, HDFS operating tools

4. Execution code and results:

Recommend.java code:

import java.util.HashMap;

import java.util.Map;

import java.util.regex.Pattern;

import org.apache.hadoop.mapred.JobConf;

public class Recommend {

public static final String HDFS = "hdfs: //10.1.32.91: 9000";

public static final Pattern DELIMITER = Pattern.compile ("[\ t,]");

public static void main (String [] args) throws Exception {

Map <String, String> path = new HashMap <String, String> ();

path.put ("data", "C: \\ Users \\ licz \\ workspace \\ Mapreduce1 \\ small2.csv");

path.put ("Step1Input", HDFS + "/ user / hdfs / recommend");

path.put ("Step1Output", path.get ("Step1Input") + "/ step1");

path.put ("Step2Input", path.get ("Step1Output"));

path.put ("Step2Output", path.get ("Step1Input") + "/ step2");

path.put ("Step3Input1", path.get ("Step1Output"));

path.put ("Step3Output1", path.get ("Step1Input") + "/ step3_1");

path.put ("Step3Input2", path.get ("Step2Output"));

path.put ("Step3Output2", path.get ("Step1Input") + "/ step3_2");

path.put ("Step4Input1", path.get ("Step3Output1"));

path.put ("Step4Input2", path.get ("Step3Output2"));

path.put ("Step4Output", path.get ("Step1Input") + "/ step4");

Step1.run (path);

Step2.run (path);

Step3.run1 (path);

Step3.run2 (path);

Step4.run (path);

System.exit (0);

}

public static JobConf config () {

JobConf conf = new JobConf (Recommend.class);

conf.setJobName ("Recommand");

conf.addResource ("classpath: /hadoop/core-site.xml");

conf.addResource ("classpath: /hadoop/hdfs-site.xml");

conf.addResource ("classpath: /hadoop/mapred-site.xml");

//conf.set("io.sort.mb "," 1024 ");

return conf;

}

}