Mahout源碼MeanShiftCanopyDriver分析之三MeanShiftCanopyReducer資料邏輯流

首先貼上MeanShiftCanopyReducer的仿造代碼，如下：

package mahout.fansy.meanshift;import java.io.IOException;import java.util.Collection;import java.util.HashMap;import java.util.Map;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.Text;import org.apache.mahout.clustering.iterator.ClusterWritable;import org.apache.mahout.clustering.meanshift.MeanShiftCanopy;import org.apache.mahout.clustering.meanshift.MeanShiftCanopyClusterer;import org.apache.mahout.clustering.meanshift.MeanShiftCanopyConfigKeys;import com.google.common.collect.Lists;public class MeanShiftCanopyReducerFollow {/** * MeanShiftCanopyReducer仿造代碼 * @author fansy * @param args *///private static int convergedClusters=0;private static boolean allConverged = true;public static void main(String[] args) {//cleanup();// 調試cleanup函數reduce(); // 調試reduce函數}/** * 仿造reduce操作 */public static Map<Text,Collection<ClusterWritable>> reduce(){Collection<MeanShiftCanopy> canopies = Lists.newArrayList();// 獲得map的輸出Collection<ClusterWritable >values =MeanShiftCanopyMapperFollow.cleanup().get(new Text("0"));MeanShiftCanopyClusterer clusterer=setup();Collection<ClusterWritable> v=Lists.newArrayList();for (ClusterWritable clusterWritable : values) {    MeanShiftCanopy canopy = (MeanShiftCanopy)clusterWritable.getValue();    clusterer.mergeCanopy(canopy.shallowCopy(), canopies);    }Map<Text,Collection<ClusterWritable>> map =new HashMap<Text,Collection<ClusterWritable>>();    for (MeanShiftCanopy canopy : canopies) {      boolean converged = clusterer.shiftToMean(canopy);      if (converged) {   //     System.out.println("Clustering"+ " Converged Clusters"+convergedClusters++);      }      allConverged = converged && allConverged;      ClusterWritable clusterWritable = new ClusterWritable();      clusterWritable.setValue(canopy);      v.add(clusterWritable);      map.put(new Text(canopy.getIdentifier()), v);  //    System.out.println("key:"+canopy.getIdentifier()+",value:"+clusterWritable.getValue().toString());    }    //    map.put(new Text(canopy.getIdentifier()), v);    return map;}/** * 仿造setup函數,直接調用MapperFollow的方法 * @return 返回經過設定值的MeanShiftCanopyClusterer */public static MeanShiftCanopyClusterer setup(){return MeanShiftCanopyMapperFollow.setup();}/** * 仿造cleanup函數 * @throws IOException  */public static void cleanup() throws IOException{//int numReducers=1; // 自己設定，這裡為了方便直接設定為1Configuration conf=getConf();// 判斷是否全部都滿足準則閾值要求，滿足，則建立檔案；if (allConverged) {      Path path = new Path(conf.get(MeanShiftCanopyConfigKeys.CONTROL_PATH_KEY));      FileSystem.get(path.toUri(), conf).createNewFile(path);    }}/** * 獲得配置過的configuration * @return */public static Configuration getConf(){String measureClassName="org.apache.mahout.common.distance.EuclideanDistanceMeasure";String kernelProfileClassName="org.apache.mahout.common.kernel.TriangularKernelProfile";double convergenceDelta=0.5;double t1=47.6;double t2=1;boolean runClustering=true;Configuration conf =new Configuration();conf.set(MeanShiftCanopyConfigKeys.DISTANCE_MEASURE_KEY, measureClassName);    conf.set(MeanShiftCanopyConfigKeys.KERNEL_PROFILE_KEY,        kernelProfileClassName);    conf.set(MeanShiftCanopyConfigKeys.CLUSTER_CONVERGENCE_KEY, String        .valueOf(convergenceDelta));   conf.set(MeanShiftCanopyConfigKeys.T1_KEY, String.valueOf(t1));    conf.set(MeanShiftCanopyConfigKeys.T2_KEY, String.valueOf(t2));    conf.set(MeanShiftCanopyConfigKeys.CLUSTER_POINTS_KEY, String        .valueOf(runClustering));    return conf;}/** * 獲得map的輸出資料,即canopies * @return Map<Text, ClusterWritable> canpies */public static Map<Text, Collection<ClusterWritable>> getMapData(){return MeanShiftCanopyMapperFollow.cleanup();}}

這裡的setup函數和之前mapper中的一樣，這裡不再分析。cleanup函數也只是當滿足準則閾值時建立一個函數而已，這裡也不再多說，主要分析reduce函數（其實主要代碼和mapper中的map+cleanup函數差不多）。

map輸出的前面3條記錄如下：

MSC-0{n=100 c=[29.942, 30.443, 30.325, 30.018, 29.887, 29.777, 29.855, 29.883, 30.128, 29.984, 29.796, 29.845, 30.436, 29.729, 29.890, 29.518, 29.546, 30.052, 30.077, 30.001, 29.837, 29.928, 30.288, 30.347, 29.785, 29.799, 29.651, 30.008, 29.938, 30.104, 29.997, 29.684, 29.949, 29.754, 30.272, 30.106, 29.883, 30.221, 29.847, 29.848, 29.843, 30.577, 29.870, 29.785, 29.923, 29.864, 30.184, 29.977, 30.321, 30.068, 30.570, 30.224, 30.240, 29.969, 30.246, 30.544, 29.862, 30.099, 29.907, 30.169] r=[3.384, 3.383, 3.494, 3.523, 3.308, 3.605, 3.315, 3.518, 3.472, 3.519, 3.350, 3.444, 3.273, 3.274, 3.400, 3.443, 3.426, 3.499, 3.154, 3.506, 3.509, 3.436, 3.484, 3.475, 3.360, 3.164, 3.460, 3.491, 3.608, 3.484, 3.477, 3.748, 3.628, 3.378, 3.327, 3.600, 3.455, 3.562, 3.534, 3.566, 3.213, 3.645, 3.615, 3.274, 3.197, 3.373, 3.595, 3.452, 3.609, 3.518, 3.262, 3.477, 3.755, 3.830, 3.494, 3.676, 3.423, 3.491, 3.641, 3.374]}

MSC-1{n=101 c=[29.890, 30.422, 30.280, 30.046, 29.891, 29.805, 29.828, 29.875, 30.133, 30.035, 29.773, 29.900, 30.441, 29.751, 29.906, 29.490, 29.508, 30.013, 30.082, 30.049, 29.815, 29.934, 30.286, 30.294, 29.828, 29.831, 29.712, 30.005, 29.977, 30.128, 30.015, 29.675, 29.963, 29.766, 30.259, 30.095, 29.855, 30.139, 29.704, 29.797, 29.808, 30.530, 29.743, 29.745, 29.883, 29.741, 30.140, 29.935, 30.271, 29.934, 30.437, 30.184, 30.180, 29.823, 30.146, 30.494, 29.767, 30.061, 29.854, 30.130] r=[3.407, 3.373, 3.506, 3.517, 3.292, 3.598, 3.310, 3.502, 3.455, 3.538, 3.341, 3.471, 3.257, 3.265, 3.387, 3.437, 3.430, 3.504, 3.139, 3.522, 3.499, 3.419, 3.466, 3.497, 3.371, 3.165, 3.496, 3.474, 3.610, 3.475, 3.464, 3.730, 3.613, 3.363, 3.313, 3.584, 3.449, 3.639, 3.797, 3.585, 3.215, 3.658, 3.818, 3.282, 3.205, 3.573, 3.605, 3.460, 3.626, 3.748, 3.507, 3.482, 3.784, 4.079, 3.616, 3.692, 3.535, 3.495, 3.663, 3.380]}

MSC-2{n=100 c=[29.942, 30.443, 30.325, 30.018, 29.887, 29.777, 29.855, 29.883, 30.128, 29.984, 29.796, 29.845, 30.436, 29.729, 29.890, 29.518, 29.546, 30.052, 30.077, 30.001, 29.837, 29.928, 30.288, 30.347, 29.785, 29.799, 29.651, 30.008, 29.938, 30.104, 29.997, 29.684, 29.949, 29.754, 30.272, 30.106, 29.883, 30.221, 29.847, 29.848, 29.843, 30.577, 29.870, 29.785, 29.923, 29.864, 30.184, 29.977, 30.321, 30.068, 30.570, 30.224, 30.240, 29.969, 30.246, 30.544, 29.862, 30.099, 29.907, 30.169] r=[3.384, 3.383, 3.494, 3.523, 3.308, 3.605, 3.315, 3.518, 3.472, 3.519, 3.350, 3.444, 3.273, 3.274, 3.400, 3.443, 3.426, 3.499, 3.154, 3.506, 3.509, 3.436, 3.484, 3.475, 3.360, 3.164, 3.460, 3.491, 3.608, 3.484, 3.477, 3.748, 3.628, 3.378, 3.327, 3.600, 3.455, 3.562, 3.534, 3.566, 3.213, 3.645, 3.615, 3.274, 3.197, 3.373, 3.595, 3.452, 3.609, 3.518, 3.262, 3.477, 3.755, 3.830, 3.494, 3.676, 3.423, 3.491, 3.641, 3.374]}

前面的準備工作做好後可以直接進入clusterer.mergeCanopy(canopy.shallowCopy(), canopies);方法來看，這裡的分析和前面的mapper中的一樣，第一行輸入是一樣的，但是第二行的輸入卻不同了，第二行的輸入和canopies(1)的norm是0.44<t1 且0.44<t2，則要進入這裡：

if (norm < t2 && (closestCoveringCanopy == null || norm < closestNorm)) {        closestNorm = norm;        closestCoveringCanopy = canopy;      }

然後接下來就應該要進入else中了，而非if中，如下：

 if (closestCoveringCanopy == null) {      canopies.add(aCanopy);    } else {      closestCoveringCanopy.merge(aCanopy, runClustering);    }

這裡的merge只是把相應的canopies(1)（這裡的1可能是其他數字，針對前面幾條資料，這裡都是1）的boundPoints、mass的值，比如當canopies(1)中有三個值的時候，mass就為4，boundPoints為[0, 1, 2, 3]。這裡的merge函數比較好理解點。

現在返回到前面的touch函數，我覺得這個函數不是很好理解；看到這個函數的詳細代碼如下：

void touch(MeanShiftCanopy canopy, double weight) {    canopy.observe(getCenter(), weight * mass);    observe(canopy.getCenter(), weight * canopy.mass);  }

調用方式為：aCanopy.touch(canopy, weight);其中，aCanopy為輸入的一條記錄，canopy為canopies(i)中的一個；

針對上面代碼中的兩條操作其實都是設定s0、s1和s2的：第一句把canopy的s0+1，由於aCanopy的mass為始終為1，所以canopy的s1就是把當前的s1+aCanopy的center，（s2的計算有點複雜，暫時不關注，其實和s1差不多，只是計算複雜點）；第二句把aCanopy（它的s0、s1、s2都是空的）的s1通過canopies(1)的center值*canopies(1)的mass值得到，s0的值設定為mass值；

其實針對上面的設定canopies(1)的s0、s1、s2我是理解的，因為在後面要使用到，但是為什麼要設定aCanopy的s0、s1、s2呢？表示不理解，後面用到aCanopy的地方也只是merge函數而已，這個函數用到的也只是aCanopy的boundPoints和mass值而已並沒有用到s0、s1和s2，所以這裡不是很理解。額，好吧，好像在add方法中也用到了aCanopy，當closesCoveringCanopy為空白的時候，即要建立一個canopies(i)的時候是有用的。

這樣在reduce的輸出中就可以輸出479個值了，和第一篇部落格中說第一次迴圈得到的reduce的輸出個數是479是一樣的，如下：

這樣MeanShiftCanopyDriver的基本分析就ok了。

分享，快樂，成長

轉載請註明出處：http://blog.csdn.net/fansy1990