Samza - Hello Samza

<!DOCTYPE html>  <html lang="en"> <head> <meta http-equiv="X-UA-Compatible" content="IE=edge"> <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no"> <meta charset="utf-8"> <title>Samza - Hello Samza</title> <link rel="apple-touch-icon-precomposed" sizes="57x57" href="/img/favicon/apple-touch-icon-57x57.png" /> <link rel="apple-touch-icon-precomposed" sizes="114x114" href="/img/favicon/apple-touch-icon-114x114.png" /> <link rel="apple-touch-icon-precomposed" sizes="72x72" href="/img/favicon/apple-touch-icon-72x72.png" /> <link rel="apple-touch-icon-precomposed" sizes="144x144" href="/img/favicon/apple-touch-icon-144x144.png" /> <link rel="apple-touch-icon-precomposed" sizes="60x60" href="/img/favicon/apple-touch-icon-60x60.png" /> <link rel="apple-touch-icon-precomposed" sizes="120x120" href="/img/favicon/apple-touch-icon-120x120.png" /> <link rel="apple-touch-icon-precomposed" sizes="76x76" href="/img/favicon/apple-touch-icon-76x76.png" /> <link rel="apple-touch-icon-precomposed" sizes="152x152" href="/img/favicon/apple-touch-icon-152x152.png" /> <link rel="icon" type="image/png" href="/img/favicon/favicon-196x196.png" sizes="196x196" /> <link rel="icon" type="image/png" href="/img/favicon/favicon-96x96.png" sizes="96x96" /> <link rel="icon" type="image/png" href="/img/favicon/favicon-32x32.png" sizes="32x32" /> <link rel="icon" type="image/png" href="/img/favicon/favicon-16x16.png" sizes="16x16" /> <link rel="icon" type="image/png" href="/img/favicon/favicon-128.png" sizes="128x128" /> <meta name="application-name" content="https://samza.apache.org" /> <meta name="msapplication-TileColor" content="#FFFFFF" /> <meta name="msapplication-TileImage" content="/img/favicon/mstile-144x144.png" /> <meta name="msapplication-square70x70logo" content="/img/favicon/mstile-70x70.png" /> <meta name="msapplication-square150x150logo" content="/img/favicon/mstile-150x150.png" /> <meta name="msapplication-wide310x150logo" content="/img/favicon/mstile-310x150.png" /> <meta name="msapplication-square310x310logo" content="/img/favicon/mstile-310x310.png" /> <link href="/css/ionicons.min.css" rel="stylesheet"> <link href="/css/google-fonts.css" rel="stylesheet"> <link href="/css/syntax.css" rel="stylesheet"/> <link rel="stylesheet" href="/css/main.new.css" /> </head> <body class="page">  <div class="main-navigation" data-plugin="menu"> <div class="main-navigation__toggle" data-menu-closed> <i class="icon ion-md-menu"></i> </div> <div class="main-navigation__toggle main-navigation__toggle--opened" data-menu-opened> <i class="icon ion-md-close"></i> </div> <div class="main-navigation__inner"> <div class="main-navigation__logo"> <a href="/"> <img class="main-navigation__logo-img" src="/img/samza-logo.png" srcset="/img/samza-logo.png 1x, /img/samza-logo@2x.png 2x" alt="Samza Logo" /> </a> </div> <div class="main-navigation__items" data-menu-opened> <a class="main-navigation__item" href="/">Home</a> <a class="main-navigation__item" href="/learn/documentation/latest/core-concepts/core-concepts.html">Docs</a> <a class="main-navigation__item" href="/powered-by/">Powered By</a> <a class="main-navigation__item" href="/startup/download/">Downloads</a> <a class="main-navigation__item" href="/blog/">Blog</a> <div class="main-navigation__item main-navigation__item--group"> <div class="main-navigation__item-group-title"> Community <i class="icon ion-md-arrow-dropdown"></i> </div> <div class="main-navigation__item-group-list"> <a class="main-navigation__item" href="/community/contact-us.html">Contact Us</a> <a class="main-navigation__item" href="/contribute/contributors-corner.html">Contributor's Corner</a> <a class="main-navigation__item" href="/community/committers.html">PMC Members and committers</a> <a class="main-navigation__item" href="/meetups/">Talks and Meetups</a> </div> </div> </div> </div> </div> <div class="container"> <div class="container__toggle"> <i class="icon ion-md-arrow-dropleft-circle container__toggle-icon"></i> <i class="icon ion-md-arrow-dropright-circle container__toggle-icon container__toggle-icon--opened"></i> </div>   <div class="side-navigation">  <div class="side-navigation__group side-navigation__group--has-nested" data-plugin="sub-menu" data-sub-menu-show-class="side-navigation__group--has-nested-visible">  <div class="side-navigation__group-title"> <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> Getting Started </div> <div class="side-navigation__group-items " data-sub-menu >  <a class="side-navigation__group-item" data-match-active="" href="/startup/quick-start/latest/">QuickStart</a> <a class="side-navigation__group-item" data-match-active="" href="/startup/code-examples/latest/">Code Examples</a>   </div>  </div>  <div class="side-navigation__group side-navigation__group--has-nested" data-plugin="sub-menu" data-sub-menu-show-class="side-navigation__group--has-nested-visible">  <div class="side-navigation__group-title"> <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> Documentation </div> <div class="side-navigation__group-items side-navigation__group-has-submenus" data-sub-menu data-documentation="/learn/documentation/latest/">    </div>  </div>  <div class="side-navigation__group side-navigation__group--has-nested" data-plugin="sub-menu" data-sub-menu-show-class="side-navigation__group--has-nested-visible">  <div class="side-navigation__group-title"> <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> Releases </div> <div class="side-navigation__group-items " data-sub-menu >  <a class="side-navigation__group-item" data-match-active="" href="/releases/1.8.0">1.8.0</a> <a class="side-navigation__group-item" data-match-active="" href="/releases/1.7.0">1.7.0</a> <a class="side-navigation__group-item" data-match-active="" href="/releases/1.6.0">1.6.0</a> <a class="side-navigation__group-item" data-match-active="" href="/releases/1.5.1">1.5.1</a> <a class="side-navigation__group-item" data-match-active="" href="/releases/1.5.0">1.5.0</a> <a class="side-navigation__group-item" data-match-active="" href="/releases/1.4.0">1.4.0</a> <a class="side-navigation__group-item" data-match-active="" href="/releases/1.3.1">1.3.1</a> <a class="side-navigation__group-item" data-match-active="" href="/releases/1.3.0">1.3.0</a> <a class="side-navigation__group-item" data-match-active="" href="/releases/1.2.0">1.2.0</a> <a class="side-navigation__group-item" data-match-active="" href="/releases/1.1.0">1.1.0</a> <a class="side-navigation__group-item" data-match-active="" href="/releases/1.0.0">1.0.0</a> <a class="side-navigation__group-item" data-match-active="" href="/releases/0.14">0.14</a> <a class="side-navigation__group-item" data-match-active="" href="/releases/0.13">0.13</a> <a class="side-navigation__group-item" data-match-active="" href="/releases/0.12">0.12</a> <a class="side-navigation__group-item" data-match-active="" href="/releases/0.11">0.11</a> <a class="side-navigation__group-item" data-match-active="" href="/releases/0.10">0.10</a>   </div>  </div>  <div class="side-navigation__group">  <a class="side-navigation__group-title" data-plugin="top-menu" data-match-active="" href="/blog/"> Blog </a>     </div>  <div class="side-navigation__group side-navigation__group--has-nested" data-plugin="sub-menu" data-sub-menu-show-class="side-navigation__group--has-nested-visible">  <div class="side-navigation__group-title"> <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> Community </div> <div class="side-navigation__group-items " data-sub-menu >  <a class="side-navigation__group-item" data-match-active="" href="/community/contact-us.html">Contact Us</a> <a class="side-navigation__group-item" data-match-active="" href="/contribute/contributors-corner.html">Contributor's Corner</a> <a class="side-navigation__group-item" data-match-active="" href="/contribute/enhancement-proposal.html">Enhancement Proposal</a> <a class="side-navigation__group-item" data-match-active="" href="/community/committers.html">PMC members & Committers</a> <a class="side-navigation__group-item" data-match-active="" href="/meetups/">Talks and Meetups</a>   </div>  </div>  <div class="side-navigation__group side-navigation__group--has-nested" data-plugin="sub-menu" data-sub-menu-show-class="side-navigation__group--has-nested-visible">  <div class="side-navigation__group-title"> <i class="side-navigation__group-title-icon icon ion-md-arrow-dropdown"></i> Case Studies </div> <div class="side-navigation__group-items " data-sub-menu >  <a class="side-navigation__group-item" data-match-active="exact" href="/case-studies/">View All</a> <hr>  <a class="side-navigation__group-item" href="/case-studies/ebay" data-match-active="">eBay</a> <a class="side-navigation__group-item" href="/case-studies/tripadvisor" data-match-active="">TripAdvisor</a> <a class="side-navigation__group-item" href="/case-studies/slack" data-match-active="">Slack</a> <a class="side-navigation__group-item" href="/case-studies/optimizely" data-match-active="">Optimizely</a> <a class="side-navigation__group-item" href="/case-studies/redfin" data-match-active="">Redfin</a> <a class="side-navigation__group-item" href="/case-studies/linkedin" data-match-active="">LinkedIn</a>  </div>  </div> </div> <div class="section"> <div class="content"> <h2>Hello Samza</h2>  <p>The <a href="https://github.com/apache/samza-hello-samza">hello-samza</a> project is an example project designed to help you run your first Samza job.</p> <h3 id="get-the-code">Get the Code</h3> <p>Check out the hello-samza project:</p> <figure class="highlight"><pre><code class="language-bash" data-lang="bash">git clone https://gitbox.apache.org/repos/asf/samza-hello-samza.git hello-samza <span class="nb">cd </span>hello-samza</code></pre></figure> <p>This project contains everything you’ll need to run your first Samza jobs.</p> <h3 id="start-a-grid">Start a Grid</h3> <p>A Samza grid usually comprises three different systems: <a href="http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/YARN.html">YARN</a>, <a href="http://kafka.apache.org/">Kafka</a>, and <a href="http://zookeeper.apache.org/">ZooKeeper</a>. The hello-samza project comes with a script called “grid” to help you setup these systems. Start by running:</p> <figure class="highlight"><pre><code class="language-bash" data-lang="bash">bin/grid bootstrap</code></pre></figure> <p>This command will download, install, and start ZooKeeper, Kafka, and YARN. It will also check out the latest version of Samza and build it. All package files will be put in a sub-directory called “deploy” inside hello-samza’s root folder.</p> <p>If you get a complaint that JAVA_HOME is not set, then you’ll need to set it to the path where Java is installed on your system.</p> <p>Once the grid command completes, you can verify that YARN is up and running by going to <a href="http://localhost:8088">http://localhost:8088</a>. This is the YARN UI.</p> <h3 id="build-a-samza-job-package">Build a Samza Job Package</h3> <p>Before you can run a Samza job, you need to build a package for it. This package is what YARN uses to deploy your jobs on the grid.</p> <p><strong>(Optional)</strong> NOTE: if you want the hello-samza jobs to run with a local Samza build (e.g., if you are a Samza developer), make sure that you run the following steps, otherwise skip them.</p> <figure class="highlight"><pre><code class="language-bash" data-lang="bash">In your hello-samza project, git checkout latest In your <span class="nb">local </span>Samza project, ./gradlew publishToMavenLocal </code></pre></figure> <p>Then, you can continue w/ the following command in hello-samza project:</p> <figure class="highlight"><pre><code class="language-bash" data-lang="bash">mvn clean package <span class="nb">mkdir</span> <span class="nt">-p</span> deploy/samza <span class="nb">tar</span> <span class="nt">-xvf</span> ./target/hello-samza-1.9.0-SNAPSHOT-dist.tar.gz <span class="nt">-C</span> deploy/samza</code></pre></figure> <h3 id="run-a-samza-job">Run a Samza Job</h3> <p>After you’ve built your Samza package, you can start a job on the grid using the run-app.sh script.</p> <figure class="highlight"><pre><code class="language-bash" data-lang="bash">deploy/samza/bin/run-app.sh <span class="nt">--config-path</span><span class="o">=</span><span class="nv">$PWD</span>/deploy/samza/config/wikipedia-feed.properties</code></pre></figure> <p>The job will consume a feed of real-time edits from Wikipedia, and produce them to a Kafka topic called “wikipedia-raw”. Give the job a minute to startup, and then tail the Kafka topic:</p> <figure class="highlight"><pre><code class="language-bash" data-lang="bash">deploy/kafka/bin/kafka-console-consumer.sh <span class="nt">--bootstrap-server</span> localhost:9092 <span class="nt">--topic</span> wikipedia-raw</code></pre></figure> <p>Pretty neat, right? Now, check out the YARN UI again (<a href="http://localhost:8088">http://localhost:8088</a>). This time around, you’ll see your Samza job is running!</p> <p>If you can not see any output from Kafka consumer, you may have connection problem. Check <a href="../../../learn/tutorials/latest/run-hello-samza-without-internet.html">here</a>.</p> <h3 id="generate-wikipedia-statistics">Generate Wikipedia Statistics</h3> <p>Let’s calculate some statistics based on the messages in the wikipedia-raw topic. Start two more jobs:</p> <figure class="highlight"><pre><code class="language-bash" data-lang="bash">deploy/samza/bin/run-app.sh <span class="nt">--config-path</span><span class="o">=</span><span class="nv">$PWD</span>/deploy/samza/config/wikipedia-parser.properties deploy/samza/bin/run-app.sh <span class="nt">--config-path</span><span class="o">=</span><span class="nv">$PWD</span>/deploy/samza/config/wikipedia-stats.properties</code></pre></figure> <p>The first job (wikipedia-parser) parses the messages in wikipedia-raw, and extracts information about the size of the edit, who made the change, etc. You can take a look at its output with:</p> <figure class="highlight"><pre><code class="language-bash" data-lang="bash">deploy/kafka/bin/kafka-console-consumer.sh <span class="nt">--bootstrap-server</span> localhost:9092 <span class="nt">--topic</span> wikipedia-edits</code></pre></figure> <p>The last job (wikipedia-stats) reads messages from the wikipedia-edits topic, and calculates counts, every ten seconds, for all edits that were made during that window. It outputs these counts to the wikipedia-stats topic.</p> <figure class="highlight"><pre><code class="language-bash" data-lang="bash">deploy/kafka/bin/kafka-console-consumer.sh <span class="nt">--bootstrap-server</span> localhost:9092 <span class="nt">--topic</span> wikipedia-stats</code></pre></figure> <p>The messages in the stats topic look like this:</p> <figure class="highlight"><pre><code class="language-json" data-lang="json"><span class="p">{</span><span class="nl">"is-talk"</span><span class="p">:</span><span class="mi">2</span><span class="p">,</span><span class="nl">"bytes-added"</span><span class="p">:</span><span class="mi">5276</span><span class="p">,</span><span class="nl">"edits"</span><span class="p">:</span><span class="mi">13</span><span class="p">,</span><span class="nl">"unique-titles"</span><span class="p">:</span><span class="mi">13</span><span class="p">}</span><span class="w"> </span><span class="p">{</span><span class="nl">"is-bot-edit"</span><span class="p">:</span><span class="mi">1</span><span class="p">,</span><span class="nl">"is-talk"</span><span class="p">:</span><span class="mi">3</span><span class="p">,</span><span class="nl">"bytes-added"</span><span class="p">:</span><span class="mi">4211</span><span class="p">,</span><span class="nl">"edits"</span><span class="p">:</span><span class="mi">30</span><span class="p">,</span><span class="nl">"unique-titles"</span><span class="p">:</span><span class="mi">30</span><span class="p">,</span><span class="nl">"is-unpatrolled"</span><span class="p">:</span><span class="mi">1</span><span class="p">,</span><span class="nl">"is-new"</span><span class="p">:</span><span class="mi">2</span><span class="p">,</span><span class="nl">"is-minor"</span><span class="p">:</span><span class="mi">7</span><span class="p">}</span><span class="w"> </span><span class="p">{</span><span class="nl">"bytes-added"</span><span class="p">:</span><span class="mi">3180</span><span class="p">,</span><span class="nl">"edits"</span><span class="p">:</span><span class="mi">19</span><span class="p">,</span><span class="nl">"unique-titles"</span><span class="p">:</span><span class="mi">19</span><span class="p">,</span><span class="nl">"is-unpatrolled"</span><span class="p">:</span><span class="mi">1</span><span class="p">,</span><span class="nl">"is-new"</span><span class="p">:</span><span class="mi">1</span><span class="p">,</span><span class="nl">"is-minor"</span><span class="p">:</span><span class="mi">3</span><span class="p">}</span><span class="w"> </span><span class="p">{</span><span class="nl">"bytes-added"</span><span class="p">:</span><span class="mi">2218</span><span class="p">,</span><span class="nl">"edits"</span><span class="p">:</span><span class="mi">18</span><span class="p">,</span><span class="nl">"unique-titles"</span><span class="p">:</span><span class="mi">18</span><span class="p">,</span><span class="nl">"is-unpatrolled"</span><span class="p">:</span><span class="mi">2</span><span class="p">,</span><span class="nl">"is-new"</span><span class="p">:</span><span class="mi">2</span><span class="p">,</span><span class="nl">"is-minor"</span><span class="p">:</span><span class="mi">3</span><span class="p">}</span></code></pre></figure> <p>If you check the YARN UI, again, you’ll see that all three jobs are now listed.</p> <h3 id="shutdown">Shutdown</h3> <p>To shutdown one of the jobs, use the same script with an extra ‘–operation=kill’ argument</p> <figure class="highlight"><pre><code class="language-bash" data-lang="bash">deploy/samza/bin/run-app.sh <span class="nt">--config-path</span><span class="o">=</span><span class="nv">$PWD</span>/deploy/samza/config/wikipedia-feed.properties <span class="nt">--operation</span><span class="o">=</span><span class="nb">kill</span></code></pre></figure> <p>After you’re done, you can clean everything up using the same grid script.</p> <figure class="highlight"><pre><code class="language-bash" data-lang="bash">bin/grid stop all</code></pre></figure> <p>Congratulations! You’ve now setup a local grid that includes YARN, Kafka, and ZooKeeper, and run a Samza job on it. Next up, check out the <a href="/learn/documentation/latest/introduction/background.html">Background</a> and <a href="/learn/documentation/latest/api/overview.html">API Overview</a> pages.</p> </div> </div> </div>   <footer> <div class="footer-inner"> <div class="side-by-side"> <div> <div class="footer__heading">Learn More</div> <div class="footer__items"> <a class="footer__item" href="/meetups/">Meetups</a> <a class="footer__item" href="/blog/">Blog</a> <a class="footer__item" href="/learn/documentation/latest/introduction/background.html">About</a> </div> </div> <div> <div class="footer__heading">Community</div> <div class="footer__items"> <a class="footer__item" href="/community/contact-us.html">Contact Us</a> <a class="footer__item" href="/contribute/contributors-corner.html">Contributors' Corner</a> <a class="footer__item" href="/community/committers.html">PMC members and committers</a> <a class="footer__item" href="/powered-by/">Powered By</a> </div> </div> <div> <div class="quick-links"> <a class="quick-link" href="/startup/download" target="_blank"> <i class="icon ion-md-download"></i> </a> <a class="quick-link" href="https://git-wip-us.apache.org/repos/asf?p=samza.git;a=tree" target="_blank"> <i class="icon ion-md-code"></i> </a> <a class="quick-link" href="https://twitter.com/samzastream" target="_blank"> <i class="icon ion-logo-twitter"></i> </a> </div> <p> <script>document.write(new Date().getFullYear());</script> © samza.apache.org</p> </div> </div> </div> </footer> <script> var tryFile = function (url, cb) { var myRequest = new Request(url); fetch(myRequest).then((response, cb) => { console.log(response.status); // returns 200 cb(response.status != 404); }); } tryFile(window.location.pathname, function (status) { // do something with the status console.log(status); }); </script>  <script> (function (i, s, o, g, r, a, m) { i['GoogleAnalyticsObject'] = r; i[r] = i[r] || function () { (i[r].q = i[r].q || []).push(arguments) }, i[r].l = 1 * new Date(); a = s.createElement(o), m = s.getElementsByTagName(o)[0]; a.async = 1; a.src = g; m.parentNode.insertBefore(a, m) })(window, document, 'script', '//www.google-analytics.com/analytics.js', 'ga'); ga('create', 'UA-43122768-1', 'apache.org'); ga('send', 'pageview'); </script> <script src="/js/main.new.js"></script> </body> </html>

CINXE.COM

Samza - Hello Samza