Yarn multipart and multipart source code analysis

public class FileSplit extends InputSplit implements Writable {private Path file;private long start;private long length;private String[] hosts;public FileSplit() {}public FileSplit(Path file, long start, long length, String[] hosts) {this.file = file;this.start = start;this.length = length;this.hosts = hosts;}public Path getPath() {return this.file;}public long getStart() {return this.start;}public long getLength() {return this.length;}public String toString() {return this.file + ":" + this.start + "+" + this.length;}public void write(DataOutput out) throws IOException {Text.writeString(out, this.file.toString());out.writeLong(this.start);out.writeLong(this.length);}public void readFields(DataInput in) throws IOException {this.file = new Path(Text.readString(in));this.start = in.readLong();this.length = in.readLong();this.hosts = null;}public String[] getLocations() throws IOException {if (this.hosts == null) {return new String[0];}return this.hosts;}}

The code is relatively simple. The four parts constitute the file path, start position, length, and host list.

Why is host a list?

Create a function when reading the parts

Splits. Add (makesplit (path, length-bytesremaining,
Splitsize, blklocations [blkindex]. gethosts ()));

Let's take a look at the source code of the block.

public class BlockLocation {    private String[] hosts;    private String[] names;    private String[] topologyPaths;    private long offset;    private long length;    private boolean corrupt;    public BlockLocation() {        this(new String[0], new String[0], 0L, 0L);    }    public BlockLocation(String[] names, String[] hosts, long offset,            long length) {        this(names, hosts, offset, length, false);    }    public BlockLocation(String[] names, String[] hosts, long offset,            long length, boolean corrupt) {        if (names == null)            this.names = new String[0];        else {            this.names = names;        }        if (hosts == null)            this.hosts = new String[0];        else {            this.hosts = hosts;        }        this.offset = offset;        this.length = length;        this.topologyPaths = new String[0];        this.corrupt = corrupt;    }    public BlockLocation(String[] names, String[] hosts,            String[] topologyPaths, long offset, long length) {        this(names, hosts, topologyPaths, offset, length, false);    }    public BlockLocation(String[] names, String[] hosts,            String[] topologyPaths, long offset, long length, boolean corrupt) {        this(names, hosts, offset, length, corrupt);        if (topologyPaths == null)            this.topologyPaths = new String[0];        else            this.topologyPaths = topologyPaths;    }    public String[] getHosts() throws IOException {        if ((this.hosts == null) || (this.hosts.length == 0)) {            return new String[0];        }        return this.hosts;    }    public String[] getNames() throws IOException {        if ((this.names == null) || (this.names.length == 0)) {            return new String[0];        }        return this.names;    }    public String[] getTopologyPaths() throws IOException {        if ((this.topologyPaths == null) || (this.topologyPaths.length == 0)) {            return new String[0];        }        return this.topologyPaths;    }    public long getOffset() {        return this.offset;    }    public long getLength() {        return this.length;    }    public boolean isCorrupt() {        return this.corrupt;    }    public void setOffset(long offset) {        this.offset = offset;    }    public void setLength(long length) {        this.length = length;    }    public void setCorrupt(boolean corrupt) {        this.corrupt = corrupt;    }    public void setHosts(String[] hosts) throws IOException {        if (hosts == null)            this.hosts = new String[0];        else            this.hosts = hosts;    }    public void setNames(String[] names) throws IOException {        if (names == null)            this.names = new String[0];        else            this.names = names;    }    public void setTopologyPaths(String[] topologyPaths) throws IOException {        if (topologyPaths == null)            this.topologyPaths = new String[0];        else            this.topologyPaths = topologyPaths;    }    public String toString() {        StringBuilder result = new StringBuilder();        result.append(this.offset);        result.append(‘,‘);        result.append(this.length);        if (this.corrupt) {            result.append("(corrupt)");        }        for (String h : this.hosts) {            result.append(‘,‘);            result.append(h);        }        return result.toString();    }}