{"id":477,"date":"2023-05-28T01:21:27","date_gmt":"2023-05-27T17:21:27","guid":{"rendered":"http:\/\/mapleleaf666.vip\/?p=477"},"modified":"2023-06-30T01:00:17","modified_gmt":"2023-06-29T17:00:17","slug":"flume%e4%b8%8ekafka%e5%af%b9%e6%8e%a5","status":"publish","type":"post","link":"http:\/\/mapleleaf666.vip\/?p=477","title":{"rendered":"flume\u4e0ekafka\u5bf9\u63a5"},"content":{"rendered":"\n<p>\u7531\u4e8e\u9879\u76ee\u9700\u6c42\uff0c\u8981\u901a\u8fc7Flume\u8bfb\u53d6\u6570\u636e\u6e90\u7684log\uff0c\u5e76\u5c06\u65e5\u5fd7\u6570\u636e\u5199\u5165Kafka\uff0c\u56e0\u6b64\u7b80\u5355\u505a\u4e86\u5bf9\u63a5\u7684\u8bb0\u5f55\u3002<\/p>\n\n\n\n<h3 class=\"wp-block-heading\">\u914d\u7f6e\u9636\u6bb5<\/h3>\n\n\n\n<p>kafka\u914d\u7f6e\u597d\uff0c\u76d1\u542c\u63a5\u53e39092<\/p>\n\n\n\n<p>flume\u914d\u7f6e\u5982\u4e0b\uff1a<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>agent.sources = r1<br>agent.sinks = k1<br>agent.channels = c1<br>\u200b<br>\u200b<br># netcat \u76d1\u542c\u7aef\u53e3<br>agent.sources.r1.type = netcat<br>agent.sources.r1.bind = localhost<br>agent.sources.r1.port = 4444<br>agent.sources.r1.channels = c1 <br># \u4e00\u884c\u7684\u6700\u5927\u5b57\u8282\u6570<br>agent.sources.r1.max-line-length = 1024000<br>\u200b<br>\u200b<br># channels\u5177\u4f53\u914d\u7f6e<br>agent.channels.c1.type = memory <br>agent.channels.c1.capacity = 1000<br>agent.channels.c1.transactionCapacity = 100<br>\u200b<br>\u200b<br># KAFKA_sinks<br>agent.sinks.k1.channel = c1<br>agent.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink<br>agent.sinks.k1.topic = hello<br>agent.sinks.k1.brokerList = localhost:9092<br>agent.sinks.k1.requiredAcks = 1<br>agent.sinks.k1.batchSize = 20<\/code><\/pre>\n\n\n\n<h3 class=\"wp-block-heading\">\u542f\u52a8\u9636\u6bb5<\/h3>\n\n\n\n<p>(\u6ca1\u6709\u8bf4\u660e\u7684\u8bdd\u76f4\u63a5\u5728\u670d\u52a1\u5668\u8f93\u5165\u547d\u4ee4\u5373\u53ef\uff0c\u65e0\u9700\u5230\u7279\u5b9a\u76ee\u5f55)<\/p>\n\n\n\n<p>\u542f\u52a8zookeeper<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>zkServer.sh start<\/code><\/pre>\n\n\n\n<p>\u542f\u52a8kafka<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>kafka-server-start.sh $KAFKA_HOME\/config\/server.properties<\/code><\/pre>\n\n\n\n<p>\u65b0\u5f00\u7a97\u53e3\u542f\u52a8\u6d88\u8d39\u8005<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic hello --from-beginning<\/code><\/pre>\n\n\n\n<p>\u542f\u52a8flume<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>\/\/\u5148\u8fdb\u5165flume\u76ee\u5f55<br>cd \/home\/ubuntu\/BI-Basic\/flume<br>\u200b<br>\/\/\u542f\u52a8\u547d\u4ee4<br>.\/bin\/flume-ng agent --conf conf\/ --name agent --conf-file conf\/flume-conf.properties -Dflume.root.logger==INFO,console<\/code><\/pre>\n\n\n\n<p>\u8bbf\u95ee\u7aef\u53e34444\uff0c\u6a21\u62df\u4fe1\u606f\u751f\u4ea7\u8005\u8f93\u5165\u4fe1\u606f<\/p>\n\n\n\n<pre class=\"wp-block-code\"><code>telnet localhost 4444<\/code><\/pre>\n\n\n\n<p>\u7ed3\u679c\u5982\u4e0b\uff1a<\/p>\n\n\n\n<p>flume\u76d1\u542c\u7684\u63a5\u53e3\u8f93\u5165\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-image\"><div class='fancybox-wrapper lazyload-container-unload' data-fancybox='post-images' href='https:\/\/s2.loli.net\/2023\/05\/22\/kwHltDp2eYJTQUg.png'><img class=\"lazyload lazyload-style-1\" src=\"data:image\/svg+xml;base64,PCEtLUFyZ29uTG9hZGluZy0tPgo8c3ZnIHdpZHRoPSIxIiBoZWlnaHQ9IjEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgc3Ryb2tlPSIjZmZmZmZmMDAiPjxnPjwvZz4KPC9zdmc+\"  decoding=\"async\" data-original=\"https:\/\/s2.loli.net\/2023\/05\/22\/kwHltDp2eYJTQUg.png\" src=\"data:image\/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsQAAA7EAZUrDhsAAAANSURBVBhXYzh8+PB\/AAffA0nNPuCLAAAAAElFTkSuQmCC\" alt=\"image-20230522093009277\"\/><\/div><\/figure>\n\n\n\n<p>kafka\u6d88\u8d39\u8005\uff1a<\/p>\n\n\n\n<figure class=\"wp-block-image\"><div class='fancybox-wrapper lazyload-container-unload' data-fancybox='post-images' href='https:\/\/s2.loli.net\/2023\/05\/22\/nfwBCaJFeNd7TUW.png'><img class=\"lazyload lazyload-style-1\" src=\"data:image\/svg+xml;base64,PCEtLUFyZ29uTG9hZGluZy0tPgo8c3ZnIHdpZHRoPSIxIiBoZWlnaHQ9IjEiIHhtbG5zPSJodHRwOi8vd3d3LnczLm9yZy8yMDAwL3N2ZyIgc3Ryb2tlPSIjZmZmZmZmMDAiPjxnPjwvZz4KPC9zdmc+\"  decoding=\"async\" data-original=\"https:\/\/s2.loli.net\/2023\/05\/22\/nfwBCaJFeNd7TUW.png\" src=\"data:image\/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAAAXNSR0IArs4c6QAAAARnQU1BAACxjwv8YQUAAAAJcEhZcwAADsQAAA7EAZUrDhsAAAANSURBVBhXYzh8+PB\/AAffA0nNPuCLAAAAAElFTkSuQmCC\" alt=\"image-20230522093031722\"\/><\/div><\/figure>\n","protected":false},"excerpt":{"rendered":"<p>\u7531\u4e8e\u9879\u76ee\u9700\u6c42\uff0c\u8981\u901a\u8fc7Flume\u8bfb\u53d6\u6570\u636e\u6e90\u7684log\uff0c\u5e76\u5c06\u65e5\u5fd7\u6570\u636e\u5199\u5165Kafka\uff0c\u56e0\u6b64\u7b80\u5355\u505a\u4e86\u5bf9\u63a5\u7684\u8bb0\u5f55\u3002 \u914d\u7f6e\u9636 [&hellip;]<\/p>\n","protected":false},"author":1,"featured_media":246,"comment_status":"open","ping_status":"open","sticky":false,"template":"","format":"standard","meta":{"footnotes":""},"categories":[11],"tags":[20,21],"class_list":["post-477","post","type-post","status-publish","format-standard","has-post-thumbnail","hentry","category-summary","tag-big-data","tag-deployment"],"_links":{"self":[{"href":"http:\/\/mapleleaf666.vip\/index.php?rest_route=\/wp\/v2\/posts\/477","targetHints":{"allow":["GET"]}}],"collection":[{"href":"http:\/\/mapleleaf666.vip\/index.php?rest_route=\/wp\/v2\/posts"}],"about":[{"href":"http:\/\/mapleleaf666.vip\/index.php?rest_route=\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"http:\/\/mapleleaf666.vip\/index.php?rest_route=\/wp\/v2\/users\/1"}],"replies":[{"embeddable":true,"href":"http:\/\/mapleleaf666.vip\/index.php?rest_route=%2Fwp%2Fv2%2Fcomments&post=477"}],"version-history":[{"count":1,"href":"http:\/\/mapleleaf666.vip\/index.php?rest_route=\/wp\/v2\/posts\/477\/revisions"}],"predecessor-version":[{"id":478,"href":"http:\/\/mapleleaf666.vip\/index.php?rest_route=\/wp\/v2\/posts\/477\/revisions\/478"}],"wp:featuredmedia":[{"embeddable":true,"href":"http:\/\/mapleleaf666.vip\/index.php?rest_route=\/wp\/v2\/media\/246"}],"wp:attachment":[{"href":"http:\/\/mapleleaf666.vip\/index.php?rest_route=%2Fwp%2Fv2%2Fmedia&parent=477"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"http:\/\/mapleleaf666.vip\/index.php?rest_route=%2Fwp%2Fv2%2Fcategories&post=477"},{"taxonomy":"post_tag","embeddable":true,"href":"http:\/\/mapleleaf666.vip\/index.php?rest_route=%2Fwp%2Fv2%2Ftags&post=477"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}