Strom入门(二)

最新推荐文章于 2023-07-31 17:43:52 发布

zero__007

最新推荐文章于 2023-07-31 17:43:52 发布

阅读量384

点赞数

CC 4.0 BY-SA版权

分类专栏： # Strom

本文链接：https://blog.csdn.net/zero__007/article/details/47703751

Strom 专栏收录该内容

15 篇文章

订阅专栏

storm有一种机制可以保证从spout发出的每个tuple都会被完全处理。当bolt处理成功时，bolt会调用collector.ack()，失败会调用collector.fail()。
这里的Spout与Blot要implements IRichSpout、IRichBolt。

示例：
SimpleSpout.java

import java.util.Map;
import java.util.Random;


import backtype.storm.spout.SpoutOutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.IRichSpout;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Fields;
import backtype.storm.tuple.Values;


/**
 * Spout起到和外界沟通的作用，他可以从一个数据库中按照某种规则取数据，也可以从分布式队列中取任务
 */
public class SimpleSpout implements IRichSpout {
	// 用来发射数据的工具类
	private SpoutOutputCollector collector;
	private int index = 0;
	private static String[] info = new String[] { "storm", "hadoop", "flume" };
	Random random = new Random();

	/**
	 * 定义字段id，该id在简单模式下没有用处，但在按照字段分组的模式下有很大的用处。
	 * 该declarer变量有很大作用，还可以调用declarer.declareStream()
	 * 来定义stramId，该id可以用来定义更加复杂的流拓扑结构
	 */
	@Override
	public void declareOutputFields(OutputFieldsDeclarer declarer) {
		declarer.declare(new Fields("string"));
	}

	/**
	 * 初始化collector
	 */
	@Override
	public void open(Map conf, TopologyContext context,
			SpoutOutputCollector collector) {
		this.collector = collector;
	}

	@Override
	public void activate() {
	}

	/**
	 * 在SpoutTracker类中被调用，每调用一次就可以向storm集群中发射一条数据（一个tuple元组），该方法会被不停的调用
	 */
	@Override
	public void nextTuple() {
		try {
			// 调用发射方法，必须指明此次发送的数据的msgId，以便ack、fail方法重发
			collector
					.emit(new Values(info[random.nextInt(info.length)]), index);
			index++;
			// 模拟等待1000ms
			Thread.sleep(1000);
		} catch (InterruptedException e) {
			e.printStackTrace();
		}
	}

	@Override
	public void ack(Object msgId) {
		System.out.println("-------sends successfully. msgId=" + msgId);
	}
	
	@Override
	public void fail(Object msgId) {
		System.out
				.println("------error : message sends unsuccessfully (msgId = "
						+ msgId + ")");
		System.out.println("------resending...");
		collector.emit(new Values(info[(int) msgId%info.length]), msgId);
		System.out.println("------resend successfully");
	}

	@Override
	public void close() {
		// TODO Auto-generated method stub
	}

	@Override
	public void deactivate() {
		// TODO Auto-generated method stub


	}

	@Override
	public Map<String, Object> getComponentConfiguration() {
		// TODO Auto-generated method stub
		return null;
	}
}

SimpleBolt.java

import java.util.Map;


import backtype.storm.task.OutputCollector;
import backtype.storm.task.TopologyContext;
import backtype.storm.topology.IRichBolt;
import backtype.storm.topology.OutputFieldsDeclarer;
import backtype.storm.tuple.Tuple;
import backtype.storm.tuple.Values;


/**
 * 接收喷发节点(Spout)发送的数据进行简单的处理后，发射出去。
 * 
 */
public class SimpleBolt implements IRichBolt {

	private int count = 1;
	private OutputCollector collector;
	
	@Override
	public void declareOutputFields(OutputFieldsDeclarer declarer) {
	}

	@Override
	public void prepare(Map stormConf, TopologyContext context,
			OutputCollector collector) {
		this.collector = collector;
	}

	@Override
	public void cleanup() {

	}

	@Override
	public void execute(Tuple input) {
		try {
			if (count % 2 == 0) {
				collector.fail(input);
			} else {
				String msg = input.getString(0);
				if (msg != null) {
					System.out.println("******" + msg);
					collector.ack(input);
				}
			}
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			count++;
		}
	}

	@Override
	public Map<String, Object> getComponentConfiguration() {
		return null;
	}

}

SimpleTopology.java

import backtype.storm.Config;
import backtype.storm.LocalCluster;
import backtype.storm.StormSubmitter;
import backtype.storm.topology.TopologyBuilder;


/**
 * 定义了一个简单的topology，包括一个数据喷发节点spout和一个数据处理节点bolt。
 * 
 */
public class SimpleTopology {
	public static void main(String[] args) {
		// 实例化TopologyBuilder类。
		TopologyBuilder topologyBuilder = new TopologyBuilder();
		// 设置喷发节点并分配并发数，该并发数将会控制该对象在集群中的线程数。
		topologyBuilder.setSpout("SimpleSpout", new SimpleSpout(), 1);
		// 设置数据处理节点并分配并发数。指定该节点接收喷发节点的策略为随机方式。
		topologyBuilder.setBolt("SimpleBolt", new SimpleBolt(), 2)
				.shuffleGrouping("SimpleSpout");

		Config config = new Config();
		config.setDebug(false);

		// 这里是本地模式下运行的启动代码。
		// config.setMaxTaskParallelism(1);
		LocalCluster cluster = new LocalCluster();
		cluster.submitTopology("simple", config,
				topologyBuilder.createTopology());
	}
}

运行结果：

。。。
------resend successfully
-------sends successfully. msgId=3
------error : message sends unsuccessfully (msgId = 4)
------resending...
------resend successfully
-------sends successfully. msgId=2
******hadoop
------error : message sends unsuccessfully (msgId = 4)
------resending...
******hadoop
------resend successfully
-------sends successfully. msgId=5
-------sends successfully. msgId=4
------error : message sends unsuccessfully (msgId = 6)
------resending...
------resend successfully
******flume
------error : message sends unsuccessfully (msgId = 6)
------resending...
******storm
------resend successfully
-------sends successfully. msgId=7
-------sends successfully. msgId=6
------error : message sends unsuccessfully (msgId = 8)
------resending...
------resend successfully
******storm
。。。