解决：
1 在构造是创建Text 类型 Tag对象。
2 由于Data 对象无法是Writeable类型，无法创建，所以只能在序列化时多记录类型，在readRields时反射出来。
详细见上代码。
(2)继承DataJoinMapperBase 实现记录标记
主要实现三个方法：
/**
* 根据文件名实现找打标记
*/
protected abstract Text generateInputTag(String inputFile);
/**
* 对本行记录打上标记，生成TaggedMapOutput
*/
protected abstract TaggedMapOutput generateTaggedMapOutput(Object value);
/**
* 生成分组key，其实就是Reduce的key
*/
protected abstract Text generateGroupKey(TaggedMapOutput aRecord);
代码：
/**
*
* 1 根据文件名作为tag，区分数据来源 2 将数据封装成TaggedMapOutput 对象，并打上必要的tag 3 生成group
* by的分组key，作为依据
*
* */
public static class JoinMapper extends DataJoinMapperBase {
/**
* 读取输入的文件路径
*
* **/
protected Text generateInputTag(String inputFile) {
// 取文件名的A和B作为来源标记
String datasource = StringUtils.splitByWholeSeparatorPreserveAllTokens(inputFile, ".", 2)[0];
return new Text(datasource);
}
/***
* 分组的Key
*
* **/
protected Text generateGroupKey(TaggedMapOutput aRecord) {
String line = ((Text) aRecord.getData()).toString();
if (StringUtils.isBlank(line)) {
return null;
}
// 去每个文件的第一个字段作为连接key
String groupKey = StringUtils.splitByWholeSeparatorPreserveAllTokens(line, ",", 2)[0];
return new Text(groupKey);
}
/**
* 对文件打上标记
*/
protected TaggedMapOutput generateTaggedMapOutput(Object value) {
TaggedWritable retv = new TaggedWritable((Text) value);
retv.setTag(this.inputTag);
return retv;
}
}(3)继承DataJoinReducerBase 根据条件数据数据
主要实现combin()方法，完成连接操作
public static class JoinReducer extends DataJoinReducerBase {
/**
* tags，标签集合，且有顺序通常按照文件读取顺序 values，标签值，
*
* 本方法被调一次，会传递一组要连接的记录，文件1的一条，文件2的一条
*/
protected TaggedMapOutput combine(Object[] tags, Object[] values) {
// 按照需求，非left join 或 right join所以要求
if (tags.length < 2)
return null;
String joinedStr = "";
for (int i = 0; i < values.length; i++) {
// 设置拼接符
if (i > 0)
joinedStr += ",";
TaggedWritable tw = (TaggedWritable) values[i];
String line = ((Text) tw.getData()).toString();
String[] tokens = line.split(",", 2);
joinedStr += tokens[1];
}
// 写出
TaggedWritable retv = new TaggedWritable(new Text(joinedStr));
retv.setTag((Text) tags[0]);
return retv;
}
}
(4)整体调用代码
public class MR1ReduceJoinJob {
public static void main(String[] args) throws Exception {
String in = "/Test/demo/in";
String out = "/Test/demo/out";
Configuration conf = new Configuration();
FileSystem fs = FileSystem.get(conf);
JobConf job = buildJob(new Path(in), new Path(out), fs, conf);
RunningJob runningJob = JobClient.runJob(job);
// 等待结束
runningJob.waitForCompletion();
if (runningJob.isSuccessful()) {
System.out.println("success ! ");
} else {
System.out.println(runningJob.getFailureInfo());
}
}
public static JobConf buildJob(Path in, Path out, FileSystem fs, Configuration conf) throws IOException {
fs.delete(out, true);
JobConf job = new JobConf(new Configuration(), MR1ReduceJoinJob.class);
job.setJobName("MR1ReduceJoinJob");
job.setJarByClass(MR1ReduceJoinJob.class);
job.setMapperClass(JoinMapper.class);
job.setReducerClass(JoinReducer.class);
job.setNumReduceTasks(1);
job.setInputFormat(TextInputFormat.class);
job.setOutputFormat(TextOutputFormat.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(TaggedWritable.class);
job.set("mapred.textoutputformat.separator", ",");
// 解决死循环问题
job.setLong("datajoin.maxNumOfValuesPerGroup", Long.MAX_VALUE);
FileInputFormat.addInputPath(job, in);
FileOutputFormat.setOutputPath(job, out);
return job;
}
}
输入文件：
A：
1,a
2,b
3,c
B：
1,11
1,111
2,22
2,222
4,44
输出文件：
1,a,111
1,a,11
2,b,222
2,b,22
其他的坑：
大文件执行是一直出这个：
2-13 17:54:45 -2924 [localfetcher#5] INFO - closeInMemoryFile -> map-output of size: 215770, inMemoryMapOutputs.size() -> 2, commitMemory -> 829095, usedMemory ->1044865
2015-12-13 17:54:45 -2925 [localfetcher#5] INFO - localfetcher#5 about to shuffle output of map attempt_local647573184_0001_m_000009_0 decomp: 205864 len: 205868 to MEMORY
2015-12-13 17:54:45 -2925 [localfetcher#5] INFO - Read 205864 bytes from map-output for attempt_local647573184_0001_m_000009_0
2015-12-13 17:54:45 -2925 [localfetcher#5] INFO - closeInMemoryFile -> map-output of size: 205864, inMemoryMapOutputs.size() -> 3, commitMemory -> 1044865, usedMemory ->1250729
2015-12-13 17:54:45 -2926 [localfetcher#5] INFO - localfetcher#5 about to shuffle output of map attempt_local647573184_0001_m_000007_0 decomp: 211843 len: 211847 to MEMORY
2015-12-13 17:54:45 -2926 [localfetcher#5] INFO - Read 211843 bytes from map-output for attempt_local647573184_0001_m_000007_0
2015-12-13 17:54:45 -2926 [localfetcher#5] INFO - closeInMemoryFile -> map-output of size: 211843, inMemoryMapOutputs.size() -> 4, commitMemory -> 1250729, usedMemory ->1462572
2015-12-13 17:54:45 -2927 [localfetcher#5] INFO - localfetcher#5 about to shuffle output of map attempt_local647573184_0001_m_000000_0 decomp: 851861 len: 851865 to MEMORY
2015-12-13 17:54:45 -2929 [localfetcher#5] INFO - Read 851861 bytes from map-output for attempt_local647573184_0001_m_000000_0
2015-12-1
根据源码及别人方案，要改成
job.setLong("datajoin.maxNumOfValuesPerGroup", Long.MAX_VALUE);后来问题又不出现了。
4 不足
- 基于mrv1 实现， API 交旧
Map的问题：
(1)只能基于文件名分组，要求连接的文件名必须能区分
(2)generateTaggedMapOutput()还需要手动绑定Tag
Reduce 问题：
(1)迭代输出时不能灵活控制，框架给分组，输出单个文件的字段会重，需要自己处理。
排序相同Group key的记录，也不适合处理超大的记录，可以通过二次排序改进。
- 我这种需要查找exist的需求不好实现。
二 基于MR V2 重写并改进
1 TaggedValue
package x.bd.hadoop.join.base;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.util.ReflectionUtils;
/**
* 可标记的结果
*
* @author shilei
*
*/
public class TaggedValue implements Writable {
private Text tag;
private Writable data;
public TaggedValue() {
tag = new Text();
}
public TaggedValue(Writable data) {
tag = new Text();
this.data = data;
}
@Override
public void write(DataOutput out) throws IOException {
// 写出内容
this.tag.write(out);
out.writeUTF(this.data.getClass().getName());
this.data.write(out);
}
@Override
public void readFields(DataInput in) throws IOException {
this.tag.readFields(in);
String dataClz = in.readUTF();
try {
if (this.data == null || !this.data.getClass().getName().equals(dataClz)) {
this.data = (Writable) ReflectionUtils.newInstance(Class.forName(dataClz), null);
}
this.data.readFields(in);
} catch (ClassNotFoundException cnfe) {
System.out.println("Problem in TaggedWritable class, method readFields.");
}
}
public Text getTag() {
return tag;
}
public void setTag(Text tag) {
this.tag = tag;
}
public Writable getData() {
return data;
}
public void setData(Writable data) {
this.data = data;
}
/**
* clone克隆 一个 对象数据
*
* @param conf
* @return
*/
public TaggedValue clone(Configuration conf) {
return (TaggedValue) WritableUtils.clone(this, conf);
}
public static void main(String[] args) {
System.out.println(TaggedValue.class.getName());
}
}
2 DataJoinMapBase
package x.bd.hadoop.join.base;
import java.io.IOException;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.lib.input.FileSplit;
/**
* 连接查询 Mapper
*
* @author shilei
*
* @param <KEYIN>
* @param <VALUEIN>
*/
public abstract class DataJoinMapperBase<KEYIN, VALUEIN> extends Mapper<KEYIN, VALUEIN, Text, TaggedValue> {
// 类型标记
protected Text inputTag;
// 输入文件路径，文件名
protected String inputFilePath, inputFileName;
/**
* 根据数据的文件名确定输入标签
*
*/
protected abstract Text generateInputTagByFile(String inputFilePath, String inputFileName);
/**
* 根据行内容处理Tag
*
* @param inputFilePath
* @param inputFileName
* @return
*/
protected Text generateInputTagByLine(Text tag, KEYIN key, VALUEIN value, Context context) {
return inputTag;
}
/**
* 封装待标签的输出
*/
protected abstract TaggedValue generateTaggedMapValue(VALUEIN value);
/**
* 生成group by的列
*/
protected abstract Text generateGroupKey(TaggedValue tagValue);
@Override
protected void setup(Mapper<KEYIN, VALUEIN, Text, TaggedValue>.Context context) throws IOException, InterruptedException {
FileSplit inputSplit = (FileSplit)context.getInputSplit();
this.inputFilePath = inputSplit.getPath().getParent().getName();
this.inputFileName = inputSplit.getPath().getName();
this.inputTag = generateInputTagByFile(this.inputFilePath, this.inputFileName);
}
@Override
public void map(KEYIN key, VALUEIN value, Context context) throws IOException, InterruptedException {
// 根据本行情况，处理Tag
this.inputTag = generateInputTagByLine(this.inputTag, key, value, context);
// 生成带标签的value
TaggedValue taggedValue = generateTaggedMapValue(value);
if (taggedValue == null) {
context.getCounter("DataJoinMapper", "discardedCount").increment(1);
return;
}
// 生成分组健
Text groupKey = generateGroupKey(taggedValue);
if (groupKey == null) {
context.getCounter("DataJoinMapper", "nullGroupKeyCount").increment(1);
return;
}
// 输出内容绑定标签
taggedValue.setTag(this.inputTag);
// key : group key , value : taggedValue
context.write(groupKey, taggedValue);
context.getCounter("DataJoinMapper", "outCount").increment(1);
}
}
3 DataJoinReduceJoin
package x.bd.hadoop.join.base;
import java.io.IOException;
import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.SortedMap;
import java.util.TreeMap;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Reducer;
/**
* 连接查询Reduce
*
* @author shilei
*
* @param <KEYOUT>
* @param <VALUEOUT>
*/
public abstract class DataJoinReducerBase<KEYOUT, VALUEOUT> extends Reducer<Text, TaggedValue, KEYOUT, VALUEOUT> {
@Override
protected void setup(Reducer<Text, TaggedValue, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException {
super.setup(context);
}
/**
* 合并结果
*/
protected abstract void combine(SortedMap<Text, List<TaggedValue>> valueGroups, Reducer<Text, TaggedValue, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException;
@Override
protected void reduce(Text key, Iterable<TaggedValue> values, Reducer<Text, TaggedValue, KEYOUT, VALUEOUT>.Context context) throws IOException, InterruptedException {
// 获得一个sort map ，key 是tab ，value 是结果的集合
SortedMap<Text, List<TaggedValue>> groups = regroup(key, values, context);
combine(groups, context);
context.getCounter("DataJoinReucer", "groupCount").increment(1);
}
/**
* 按照Tag 对value 进行充分组
*
* @param key
* @param arg1
* @param reporter
* @return
* @throws IOException
*/
private SortedMap<Text, List<TaggedValue>> regroup(Text key, Iterable<TaggedValue> values, Reducer<Text, TaggedValue, KEYOUT, VALUEOUT>.Context context) throws IOException {
/*
* key： tag; value ： TaggedValue
*/
SortedMap<Text, List<TaggedValue>> valueGroup = new TreeMap<Text, List<TaggedValue>>();
// 遍历Value
Iterator<TaggedValue> iter = values.iterator();
while (iter.hasNext()) {
// TODO 为什么需要克隆？
TaggedValue taggedValue = ((TaggedValue) iter.next()).clone(context.getConfiguration());
// 获得记录的 tag
Text tag = taggedValue.getTag();
// 从map 中获取一个iterator，如果已经创建，就做一个情况
List<TaggedValue> datas = valueGroup.get(tag);
if (datas == null) {
datas = new LinkedList<TaggedValue>();
valueGroup.put(tag, datas);
}
datas.add(taggedValue);
// System.out.println("reduce : " + taggedValue + "|" +
// tag.toString() + "|" + taggedValue.getData().toString());
taggedValue = null;
}
return valueGroup;
}
}
4 整体调用
package x.bd.hadoop.join;
import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import java.util.SortedMap;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.NullWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import x.bd.hadoop.join.base.DataJoinMapperBase;
import x.bd.hadoop.join.base.DataJoinReducerBase;
import x.bd.hadoop.join.base.TaggedValue;
/**
* 使用Hadoop API 对数据进行 Reduce 连接</br>
*
* 文件1：A.txt 1,a</br> 2,b </br> 3,c
*
* 文件2：B.txt 1,11</br> 1,111</br> 2,22</br> 2,222</br>4,44
*
* 关联查询(要求inner join)： 1,a,11</br> 1,a,111</br> 2,b,22 </br> 2,b,222</br>
*
*
* @author shilei
*
*/
public class MR2SelfReduceJoinJob extends Configured implements Tool {
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
int res = ToolRunner.run(conf, new MR2SelfReduceJoinJob(), args);
if (res == 0) {
System.out.println("MR2SelfReduceJoinJob success !");
} else {
System.out.println("MR2SelfReduceJoinJob error ! ");
}
System.exit(res);
}
@Override
public int run(String[] args) throws Exception {
String in = "/Test/demo/in";
String out = "/Test/demo/out";
Path inPath = new Path(in);
Path outPath = new Path(out);
Configuration conf = getConf();
FileSystem fs = FileSystem.get(conf);
fs.delete(outPath, true);
Job job = Job.getInstance(conf, "MR2SelfReduceJoinJob");
job.setJarByClass(MR2SelfReduceJoinJob.class);
job.setMapperClass(JoinMapper.class);
job.setReducerClass(JoinReducer.class);
job.setNumReduceTasks(1);
// 处理map的输出
job.setMapOutputKeyClass(Text.class);
job.setMapOutputValueClass(TaggedValue.class);
job.setOutputKeyClass(Writable.class);
job.setOutputValueClass(NullWritable.class);
FileInputFormat.addInputPath(job, inPath);
FileOutputFormat.setOutputPath(job, outPath);
if (job.waitForCompletion(true)) {
return 0;
} else {
return 1;
}
}
// Map=======================================
/**
*
* 1 根据文件名作为tag，区分数据来源 2 将数据封装成TaggedMapOutput 对象，并打上必要的tag 3 生成group
* by的分组key，作为依据
*
* */
public static class JoinMapper extends DataJoinMapperBase<Object, Text> {
/**
* 读取输入的文件路径
*
*/
@Override
protected Text generateInputTagByFile(String inputFilePath, String inputFileName) {
// 取文件名的A和B作为来源标记
String datasource = StringUtils.splitByWholeSeparatorPreserveAllTokens(inputFileName, ".", 2)[0];
return new Text(datasource);
}
/**
* 按需峰值要处理的记录，这里只需要原样输出
*/
@Override
protected TaggedValue generateTaggedMapValue(Text value) {
return new TaggedValue(value);
}
/**
* 数据的第一个字段作为分组key
*/
@Override
protected Text generateGroupKey(TaggedValue tagValue) {
String line = ((Text) tagValue.getData()).toString();
if (StringUtils.isBlank(line)) {
return null;
}
// 去每个文件的第一个字段作为连接key
String groupKey = StringUtils.splitByWholeSeparatorPreserveAllTokens(line, ",", 2)[0];
return new Text(groupKey);
}
}
// Reduce============================================
public static class JoinReducer extends DataJoinReducerBase<Writable, NullWritable> {
private Text key = new Text("B");
@Override
protected void combine(SortedMap<Text, List<TaggedValue>> valueGroups, Reducer<Text, TaggedValue, Writable, NullWritable>.Context context) throws IOException, InterruptedException {
// 必须能连接上
if (valueGroups.size() < 2) {
return;
}
// 这里只输出文件2的字段
// 写出
List<TaggedValue> cookieValues = valueGroups.get(key);
Iterator<TaggedValue> iter = cookieValues.iterator();
while (iter.hasNext()) {
TaggedValue value = iter.next();
if (value == null) {
continue;
}
context.write(value.getData(), NullWritable.get());
}
}
}
}三 源码包
昨天，我看到 
图1 截止11月全国各省正常平台和问题平台数量统计
图2 问题平台状态比例
图3 典型网贷诈骗流程图
图4 诈骗平台通过极高利率吸引投资者
图5 高利率、高奖励的借款项目
图6 虚标或拆标的项目
图7P2P平台存在的一些安全性问题
图8部分P2P网站漏洞类型分布
图9P2P手机应用安全问题类型分布
图10某P2P手机应用明文传输密码及金额
图11某P2P手机应用本地验证短信校验码
图12 2015年每月新增P2P理财钓鱼网站数量
图13 十月和十一月P2P理财钓鱼诈骗网站访问量
图14套用同一个模板的理财诈骗网站
图15诈骗网站高利率的虚假项目
图16某P2P曝光群曝光的某诈骗平台的虚假注册信息
图17宏量财富跑路爆料帖
