Hadoop source code analysis (2) -- configuration class

This article mainly introduces the system configuration of hadoop.

 

Next, we will introduce the main method of hadoop job in the previous article as follows:

public static void main(String[] args) throws Exception {     int res = ToolRunner.run(new Configuration(), new CalculateSumJob(),args);     System.exit(res);}

Toolrunner. the first variable passed in by the run method is new configuration. apache. hadoop. find the configuration class in the conf package, which is mainly used to configure the job parameters according to the corresponding configuration items in the configuration file. The configuration class implements the iterable interface in Java. Several key points of the configuration class are described as follows:

1. adddefaultresource ()

  public static synchronized void addDefaultResource(String name) {    if(!defaultResources.contains(name)) {      defaultResources.add(name);      for(Configuration conf : REGISTRY.keySet()) {        if(conf.loadDefaults) {          conf.reloadConfiguration();        }      }    }  }

This method loads the default configuration file name into the defaultresources list, the default configuration file is the hadoop-default.xml.

2. The loadresources () parameter is arraylist.

private void loadResources(Properties properties,                             ArrayList resources,                             boolean quiet) {    if(loadDefaults) {      for (String resource : defaultResources) {        loadResource(properties, resource, quiet);      }          //support the hadoop-site.xml as a deprecated case      if(getResource("hadoop-site.xml")!=null) {        loadResource(properties, "hadoop-site.xml", quiet);      }    }    else{         for (Object resource : resources) {            loadResource(properties, resource, quiet);         }    }  }

The method first judges loaddefaults this bool variable, the default is true, then call the default configuration file, otherwise call the specified configuration file, including core-default.xml, core-site.xml, hdfs-site.xml, mapred-site.xml, etc.
3. The loadresource () parameter is obeject.

 private void loadResource(Properties properties, Object name, boolean quiet) {    try {      DocumentBuilderFactory docBuilderFactory         = DocumentBuilderFactory.newInstance();      //ignore all comments inside the xml file      docBuilderFactory.setIgnoringComments(true);      //allow includes in the xml file      docBuilderFactory.setNamespaceAware(true);      try {          docBuilderFactory.setXIncludeAware(true);      } catch (UnsupportedOperationException e) {        LOG.error("Failed to set setXIncludeAware(true) for parser "                + docBuilderFactory                + ":" + e,                e);      }      DocumentBuilder builder = docBuilderFactory.newDocumentBuilder();      Document doc = null;      Element root = null;      if (name instanceof URL) {                  // an URL resource        URL url = (URL)name;        if (url != null) {          if (!quiet) {            LOG.info("parsing " + url);          }          doc = builder.parse(url.toString());        }      } else if (name instanceof String) {        // a CLASSPATH resource        URL url = getResource((String)name);        if (url != null) {          if (!quiet) {            LOG.info("parsing " + url);          }          doc = builder.parse(url.toString());        }      } else if (name instanceof Path) {          // a file resource        // Can‘t use FileSystem API or we get an infinite loop        // since FileSystem uses Configuration API.  Use java.io.File instead.        File file = new File(((Path)name).toUri().getPath())          .getAbsoluteFile();        if (file.exists()) {          if (!quiet) {            LOG.info("parsing " + file);          }          InputStream in = new BufferedInputStream(new FileInputStream(file));          try {            doc = builder.parse(in);          } finally {            in.close();          }        }      } else if (name instanceof InputStream) {        try {          doc = builder.parse((InputStream)name);        } finally {          ((InputStream)name).close();        }      } else if (name instanceof Element) {        root = (Element)name;      }      if (doc == null && root == null) {        if (quiet)          return;        throw new RuntimeException(name + " not found");      }      if (root == null) {        root = doc.getDocumentElement();      }      if (!"configuration".equals(root.getTagName()))        LOG.fatal("bad conf file: top-level element not <configuration>");      NodeList props = root.getChildNodes();      for (int i = 0; i < props.getLength(); i++) {        Node propNode = props.item(i);        if (!(propNode instanceof Element))          continue;        Element prop = (Element)propNode;        if ("configuration".equals(prop.getTagName())) {          loadResource(properties, prop, quiet);          continue;        }        if (!"property".equals(prop.getTagName()))          LOG.warn("bad conf file: element not <property>");        NodeList fields = prop.getChildNodes();        String attr = null;        String value = null;        boolean finalParameter = false;        for (int j = 0; j < fields.getLength(); j++) {          Node fieldNode = fields.item(j);          if (!(fieldNode instanceof Element))            continue;          Element field = (Element)fieldNode;          if ("name".equals(field.getTagName()) && field.hasChildNodes())            attr = ((Text)field.getFirstChild()).getData().trim();          if ("value".equals(field.getTagName()) && field.hasChildNodes())            value = ((Text)field.getFirstChild()).getData();          if ("final".equals(field.getTagName()) && field.hasChildNodes())            finalParameter = "true".equals(((Text)field.getFirstChild()).getData());        }                // Ignore this parameter if it has already been marked as ‘final‘        if (attr != null) {          if (value != null) {            if (!finalParameters.contains(attr)) {              properties.setProperty(attr, value);              if (storeResource) {                updatingResource.put(attr, name.toString());              }            } else if (!value.equals(properties.getProperty(attr))) {              LOG.warn(name+":a attempt to override final parameter: "+attr                     +";  Ignoring.");            }          }          if (finalParameter) {            finalParameters.add(attr);          }        }      }            } catch (IOException e) {      LOG.fatal("error parsing conf file: " + e);      throw new RuntimeException(e);    } catch (DOMException e) {      LOG.fatal("error parsing conf file: " + e);      throw new RuntimeException(e);    } catch (SAXException e) {      LOG.fatal("error parsing conf file: " + e);      throw new RuntimeException(e);    } catch (ParserConfigurationException e) {      LOG.fatal("error parsing conf file: " + e);      throw new RuntimeException(e);    }  }

This method reads the configuration file information into the properties global variable.

4. getprops ()

private synchronized Properties getProps() {    if (properties == null) {      properties = new Properties();      loadResources(properties, resources, quietmode);      if (overlay!= null) {        properties.putAll(overlay);        if (storeResource) {          for (Map.Entry<Object,Object> item: overlay.entrySet()) {            updatingResource.put((String) item.getKey(), "Unknown");          }        }      }    }    return properties;  }

If the configuration items in properties. putall (overlay) properties in this method also exist in overlay, overlay configuration items are overwritten. If configuration item information is required for the entire mapreduce task, the corresponding parameter values are directly read from properties.