CINXE.COM

<!doctype html><html lang="en"><head><title data-rh="true">The Billion Events Infrastructure | by Tilen Kavcic | Outfit7 | Medium</title><meta data-rh="true" charset="utf-8"/><meta data-rh="true" name="viewport" content="width=device-width,minimum-scale=1,initial-scale=1,maximum-scale=1"/><meta data-rh="true" name="theme-color" content="#000000"/><meta data-rh="true" name="twitter:app:name:iphone" content="Medium"/><meta data-rh="true" name="twitter:app:id:iphone" content="828256236"/><meta data-rh="true" property="al:ios:app_name" content="Medium"/><meta data-rh="true" property="al:ios:app_store_id" content="828256236"/><meta data-rh="true" property="al:android:package" content="com.medium.reader"/><meta data-rh="true" property="fb:app_id" content="542599432471018"/><meta data-rh="true" property="og:site_name" content="Medium"/><meta data-rh="true" property="og:type" content="article"/><meta data-rh="true" property="article:published_time" content="2022-12-14T08:39:00.356Z"/><meta data-rh="true" name="title" content="The Billion Events Infrastructure | by Tilen Kavcic | Outfit7 | Medium"/><meta data-rh="true" property="og:title" content="The Billion Events Infrastructure"/><meta data-rh="true" property="al:android:url" content="medium://p/c5fa1610d786"/><meta data-rh="true" property="al:ios:url" content="medium://p/c5fa1610d786"/><meta data-rh="true" property="al:android:app_name" content="Medium"/><meta data-rh="true" name="description" content="TL;DR: Outfit7 is, at its core, a data-driven company. Collecting and analyzing performance metrics from our games holds a lot of interesting challenges. In this article, I’ll explore our…"/><meta data-rh="true" property="og:description" content="Outfit7 is, at its core, a data-driven company. Collecting and analyzing performance metrics from our games holds a lot of…"/><meta data-rh="true" property="og:url" content="https://medium.com/outfit7/the-billion-events-infrastructure-c5fa1610d786"/><meta data-rh="true" property="al:web:url" content="https://medium.com/outfit7/the-billion-events-infrastructure-c5fa1610d786"/><meta data-rh="true" property="og:image" content="https://miro.medium.com/v2/da:true/resize:fit:1200/0*YpS6May9aFFtbR0D"/><meta data-rh="true" property="article:author" content="https://medium.com/@tilen.kavcic_63182"/><meta data-rh="true" name="author" content="Tilen Kavcic"/><meta data-rh="true" name="robots" content="index,noarchive,follow,max-image-preview:large"/><meta data-rh="true" name="referrer" content="unsafe-url"/><meta data-rh="true" property="twitter:title" content="The Billion Events Infrastructure"/><meta data-rh="true" name="twitter:site" content="@Medium"/><meta data-rh="true" name="twitter:app:url:iphone" content="medium://p/c5fa1610d786"/><meta data-rh="true" property="twitter:description" content="Outfit7 is, at its core, a data-driven company. Collecting and analyzing performance metrics from our games holds a lot of…"/><meta data-rh="true" name="twitter:image:src" content="https://miro.medium.com/v2/da:true/resize:fit:1200/0*YpS6May9aFFtbR0D"/><meta data-rh="true" name="twitter:card" content="summary_large_image"/><meta data-rh="true" name="twitter:label1" content="Reading time"/><meta data-rh="true" name="twitter:data1" content="5 min read"/><link data-rh="true" rel="icon" href="https://miro.medium.com/v2/5d8de952517e8160e40ef9841c781cdc14a5db313057fa3c3de41c6f5b494b19"/><link data-rh="true" rel="search" type="application/opensearchdescription+xml" title="Medium" href="/osd.xml"/><link data-rh="true" rel="apple-touch-icon" sizes="152x152" href="https://miro.medium.com/v2/resize:fill:304:304/10fd5c419ac61637245384e7099e131627900034828f4f386bdaa47a74eae156"/><link data-rh="true" rel="apple-touch-icon" sizes="120x120" href="https://miro.medium.com/v2/resize:fill:240:240/10fd5c419ac61637245384e7099e131627900034828f4f386bdaa47a74eae156"/><link data-rh="true" rel="apple-touch-icon" sizes="76x76" href="https://miro.medium.com/v2/resize:fill:152:152/10fd5c419ac61637245384e7099e131627900034828f4f386bdaa47a74eae156"/><link data-rh="true" rel="apple-touch-icon" sizes="60x60" href="https://miro.medium.com/v2/resize:fill:120:120/10fd5c419ac61637245384e7099e131627900034828f4f386bdaa47a74eae156"/><link data-rh="true" rel="mask-icon" href="https://miro.medium.com/v2/resize:fill:1000:1000/7*GAOKVe--MXbEJmV9230oOQ.png" color="#171717"/><link data-rh="true" rel="preconnect" href="https://glyph.medium.com" crossOrigin=""/><link data-rh="true" id="glyph_preload_link" rel="preload" as="style" type="text/css" href="https://glyph.medium.com/css/unbound.css"/><link data-rh="true" id="glyph_link" rel="stylesheet" type="text/css" href="https://glyph.medium.com/css/unbound.css"/><link data-rh="true" rel="author" href="https://medium.com/@tilen.kavcic_63182"/><link data-rh="true" rel="canonical" href="https://medium.com/outfit7/the-billion-events-infrastructure-c5fa1610d786"/><link data-rh="true" rel="alternate" href="android-app://com.medium.reader/https/medium.com/p/c5fa1610d786"/><script data-rh="true" type="application/ld+json">{"@context":"http:\u002F\u002Fschema.org","@type":"NewsArticle","image":["https:\u002F\u002Fmiro.medium.com\u002Fv2\u002Fda:true\u002Fresize:fit:1200\u002F0*YpS6May9aFFtbR0D"],"url":"https:\u002F\u002Fmedium.com\u002Foutfit7\u002Fthe-billion-events-infrastructure-c5fa1610d786","dateCreated":"2022-12-13T14:32:10.838Z","datePublished":"2022-12-13T14:32:10.838Z","dateModified":"2022-12-14T09:50:07.042Z","headline":"The Billion Events Infrastructure - Outfit7 - Medium","name":"The Billion Events Infrastructure - Outfit7 - Medium","description":"TL;DR: Outfit7 is, at its core, a data-driven company. Collecting and analyzing performance metrics from our games holds a lot of interesting challenges. In this article, I’ll explore our…","identifier":"c5fa1610d786","author":{"@type":"Person","name":"Tilen Kavcic","url":"https:\u002F\u002Fmedium.com\u002F@tilen.kavcic_63182"},"creator":["Tilen Kavcic"],"publisher":{"@type":"Organization","name":"Outfit7","url":"https:\u002F\u002Fmedium.com\u002Foutfit7","logo":{"@type":"ImageObject","width":60,"height":60,"url":"https:\u002F\u002Fmiro.medium.com\u002Fv2\u002Fresize:fit:120\u002F1*nTxl_7KgWQYo8PfVROUK_A.png"}},"mainEntityOfPage":"https:\u002F\u002Fmedium.com\u002Foutfit7\u002Fthe-billion-events-infrastructure-c5fa1610d786"}</script><style type="text/css" data-fela-rehydration="546" data-fela-type="STATIC">html{box-sizing:border-box;-webkit-text-size-adjust:100%}*, *:before, *:after{box-sizing:inherit}body{margin:0;padding:0;text-rendering:optimizeLegibility;-webkit-font-smoothing:antialiased;color:rgba(0,0,0,0.8);position:relative;min-height:100vh}h1, h2, h3, h4, h5, h6, dl, dd, ol, ul, menu, figure, blockquote, p, pre, form{margin:0}menu, ol, ul{padding:0;list-style:none;list-style-image:none}main{display:block}a{color:inherit;text-decoration:none}a, button, input{-webkit-tap-highlight-color:transparent}img, svg{vertical-align:middle}button{background:transparent;overflow:visible}button, input, optgroup, select, textarea{margin:0}:root{--reach-tabs:1;--reach-menu-button:1}#speechify-root{font-family:Sohne, sans-serif}div[data-popper-reference-hidden="true"]{visibility:hidden;pointer-events:none}.grecaptcha-badge{visibility:hidden} /*XCode style (c) Angel Garcia <angelgarcia.mail@gmail.com>*/.hljs {background: #fff;color: black; }/* Gray DOCTYPE selectors like WebKit */ .xml .hljs-meta {color: #c0c0c0; }.hljs-comment, .hljs-quote {color: #007400; }.hljs-tag, .hljs-attribute, .hljs-keyword, .hljs-selector-tag, .hljs-literal, .hljs-name {color: #aa0d91; }.hljs-variable, .hljs-template-variable {color: #3F6E74; }.hljs-code, .hljs-string, .hljs-meta .hljs-string {color: #c41a16; }.hljs-regexp, .hljs-link {color: #0E0EFF; }.hljs-title, .hljs-symbol, .hljs-bullet, .hljs-number {color: #1c00cf; }.hljs-section, .hljs-meta {color: #643820; }.hljs-title.class_, .hljs-class .hljs-title, .hljs-type, .hljs-built_in, .hljs-params {color: #5c2699; }.hljs-attr {color: #836C28; }.hljs-subst {color: #000; }.hljs-formula {background-color: #eee;font-style: italic; }.hljs-addition {background-color: #baeeba; }.hljs-deletion {background-color: #ffc8bd; }.hljs-selector-id, .hljs-selector-class {color: #9b703f; }.hljs-doctag, .hljs-strong {font-weight: bold; }.hljs-emphasis {font-style: italic; } </style><style type="text/css" data-fela-rehydration="546" data-fela-type="KEYFRAME">@-webkit-keyframes k1{0%{opacity:0.8}50%{opacity:0.5}100%{opacity:0.8}}@-moz-keyframes k1{0%{opacity:0.8}50%{opacity:0.5}100%{opacity:0.8}}@keyframes k1{0%{opacity:0.8}50%{opacity:0.5}100%{opacity:0.8}}</style><style type="text/css" data-fela-rehydration="546" data-fela-type="RULE">.a{font-family:medium-content-sans-serif-font, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Oxygen, Ubuntu, Cantarell, "Open Sans", "Helvetica Neue", sans-serif}.b{font-weight:400}.c{background-color:rgba(255, 255, 255, 1)}.l{display:block}.m{position:sticky}.n{top:0}.o{z-index:500}.p{padding:0 24px}.q{align-items:center}.r{border-bottom:solid 1px #F2F2F2}.y{height:41px}.z{line-height:20px}.ab{display:flex}.ac{height:57px}.ae{flex:1 0 auto}.af{color:inherit}.ag{fill:inherit}.ah{font-size:inherit}.ai{border:inherit}.aj{font-family:inherit}.ak{letter-spacing:inherit}.al{font-weight:inherit}.am{padding:0}.an{margin:0}.ao{cursor:pointer}.ap:disabled{cursor:not-allowed}.aq:disabled{color:#6B6B6B}.ar:disabled{fill:#6B6B6B}.au{width:auto}.av path{fill:#242424}.aw{height:25px}.ax{margin-left:16px}.ay{border:none}.az{border-radius:20px}.ba{width:240px}.bb{background:#F9F9F9}.bc path{fill:#6B6B6B}.be{outline:none}.bf{font-family:sohne, "Helvetica Neue", Helvetica, Arial, sans-serif}.bg{font-size:14px}.bh{width:100%}.bi{padding:10px 20px 10px 0}.bj{background-color:transparent}.bk{color:#242424}.bl::placeholder{color:#6B6B6B}.bm{display:inline-block}.bn{margin-left:12px}.bo{margin-right:12px}.bp{border-radius:4px}.bq{margin-left:24px}.br{height:24px}.bx{background-color:#F9F9F9}.by{border-radius:50%}.bz{height:32px}.ca{width:32px}.cb{justify-content:center}.ch{max-width:680px}.ci{min-width:0}.cj{animation:k1 1.2s ease-in-out infinite}.ck{height:100vh}.cl{margin-bottom:16px}.cm{margin-top:48px}.cn{align-items:flex-start}.co{flex-direction:column}.cp{justify-content:space-between}.cq{margin-bottom:24px}.cw{width:80%}.cx{background-color:#F2F2F2}.dd{height:44px}.de{width:44px}.df{margin:auto 0}.dg{margin-bottom:4px}.dh{height:16px}.di{width:120px}.dj{width:80px}.dp{margin-bottom:8px}.dq{width:96%}.dr{width:98%}.ds{width:81%}.dt{margin-left:8px}.du{color:#6B6B6B}.dv{font-size:13px}.dw{height:100%}.ep{color:#FFFFFF}.eq{fill:#FFFFFF}.er{background:rgba(133, 132, 132, 1)}.es{border-color:rgba(133, 132, 132, 1)}.ew:disabled{cursor:inherit !important}.ex:disabled{opacity:0.3}.ey:disabled:hover{background:rgba(133, 132, 132, 1)}.ez:disabled:hover{border-color:rgba(133, 132, 132, 1)}.fa{border-radius:99em}.fb{border-width:1px}.fc{border-style:solid}.fd{box-sizing:border-box}.fe{text-decoration:none}.ff{text-align:center}.fi{margin-right:32px}.fj{position:relative}.fk{fill:#6B6B6B}.fn{background:transparent}.fo svg{margin-left:4px}.fp svg{fill:#6B6B6B}.fr{box-shadow:inset 0 0 0 1px rgba(0, 0, 0, 0.05)}.fs{position:absolute}.fz{margin:0 24px}.gd{background:rgba(255, 255, 255, 1)}.ge{border:1px solid #F2F2F2}.gf{box-shadow:0 1px 4px #F2F2F2}.gg{max-height:100vh}.gh{overflow-y:auto}.gi{left:0}.gj{top:calc(100vh + 100px)}.gk{bottom:calc(100vh + 100px)}.gl{width:10px}.gm{pointer-events:none}.gn{word-break:break-word}.go{word-wrap:break-word}.gp:after{display:block}.gq:after{content:""}.gr:after{clear:both}.gs{line-height:1.23}.gt{letter-spacing:0}.gu{font-style:normal}.gv{font-weight:700}.ia{align-items:baseline}.ib{width:48px}.ic{height:48px}.id{border:2px solid rgba(255, 255, 255, 1)}.ie{z-index:0}.if{box-shadow:none}.ig{border:1px solid rgba(0, 0, 0, 0.05)}.ih{margin-left:-12px}.ii{width:28px}.ij{height:28px}.ik{z-index:1}.il{width:24px}.im{margin-bottom:2px}.in{flex-wrap:nowrap}.io{font-size:16px}.ip{line-height:24px}.ir{margin:0 8px}.is{display:inline}.it{color:rgba(133, 132, 132, 1)}.iu{fill:rgba(133, 132, 132, 1)}.ix{flex:0 0 auto}.ja{flex-wrap:wrap}.jd{white-space:pre-wrap}.je{margin-right:4px}.jf{overflow:hidden}.jg{max-height:20px}.jh{text-overflow:ellipsis}.ji{display:-webkit-box}.jj{-webkit-line-clamp:1}.jk{-webkit-box-orient:vertical}.jl{word-break:break-all}.jn{padding-left:8px}.jo{padding-right:8px}.kp> *{flex-shrink:0}.kq{overflow-x:scroll}.kr::-webkit-scrollbar{display:none}.ks{scrollbar-width:none}.kt{-ms-overflow-style:none}.ku{width:74px}.kv{flex-direction:row}.kw{z-index:2}.kz{-webkit-user-select:none}.la{border:0}.lb{fill:rgba(117, 117, 117, 1)}.le{outline:0}.lf{user-select:none}.lg> svg{pointer-events:none}.lp{cursor:progress}.lq{opacity:1}.lr{padding:4px 0}.lu{margin-top:0px}.lv{width:16px}.lx{display:inline-flex}.md{max-width:100%}.me{padding:8px 2px}.mf svg{color:#6B6B6B}.mw{line-height:1.58}.mx{letter-spacing:-0.004em}.my{font-family:source-serif-pro, Georgia, Cambria, "Times New Roman", Times, serif}.nt{margin-bottom:-0.46em}.nu{font-style:italic}.nv{margin-left:auto}.nw{margin-right:auto}.nx{max-width:1600px}.od{clear:both}.of{cursor:zoom-in}.og{z-index:auto}.oi{height:auto}.oj{line-height:1.12}.ok{letter-spacing:-0.022em}.ol{font-weight:600}.pg{margin-bottom:-0.28em}.pm{max-width:861px}.pn{list-style-type:decimal}.po{margin-left:30px}.pp{padding-left:0px}.pv{line-height:1.18}.qj{margin-bottom:-0.31em}.qk{box-shadow:inset 3px 0 0 0 #242424}.ql{padding-left:23px}.qm{margin-left:-20px}.qn{margin-bottom:26px}.qo{margin-top:6px}.qp{margin-top:8px}.qq{margin-right:8px}.qr{padding:8px 16px}.qs{border-radius:100px}.qt{transition:background 300ms ease}.qv{white-space:nowrap}.qw{border-top:none}.qx{margin-bottom:14px}.qy{height:52px}.qz{max-height:52px}.ra{box-sizing:content-box}.rb{position:static}.rd{max-width:155px}.rj{margin-right:20px}.rp{height:0px}.rq{margin-bottom:40px}.rr{margin-bottom:48px}.sf{border-radius:2px}.sh{height:64px}.si{width:64px}.sj{align-self:flex-end}.sk{flex:1 1 auto}.sq{padding-right:4px}.sr{font-weight:500}.sy{margin-top:16px}.sz{color:rgba(255, 255, 255, 1)}.ta{fill:rgba(255, 255, 255, 1)}.tb{background:rgba(25, 25, 25, 1)}.tc{border-color:rgba(25, 25, 25, 1)}.tf:disabled{opacity:0.1}.tg:disabled:hover{background:rgba(25, 25, 25, 1)}.th:disabled:hover{border-color:rgba(25, 25, 25, 1)}.ti{margin-bottom:54px}.to{gap:18px}.tp{fill:rgba(61, 61, 61, 1)}.tw{border-bottom:solid 1px #E5E5E5}.tx{margin-top:72px}.ty{padding:24px 0}.tz{margin-bottom:0px}.ua{margin-right:16px}.as:hover:not(:disabled){color:rgba(25, 25, 25, 1)}.at:hover:not(:disabled){fill:rgba(25, 25, 25, 1)}.et:hover{background:rgba(115, 113, 113, 1)}.eu:hover{border-color:rgba(115, 113, 113, 1)}.ev:hover{cursor:pointer}.fl:hover{color:#242424}.fm:hover{fill:#242424}.fq:hover svg{fill:#242424}.ft:hover{background-color:rgba(0, 0, 0, 0.1)}.iq:hover{text-decoration:underline}.iv:hover:not(:disabled){color:rgba(115, 113, 113, 1)}.iw:hover:not(:disabled){fill:rgba(115, 113, 113, 1)}.ld:hover{fill:rgba(8, 8, 8, 1)}.ls:hover{fill:#000000}.lt:hover p{color:#000000}.lw:hover{color:#000000}.mg:hover svg{color:#000000}.qu:hover{background-color:#F2F2F2}.sg:hover{background-color:none}.td:hover{background:#000000}.te:hover{border-color:#242424}.tq:hover{fill:rgba(25, 25, 25, 1)}.bd:focus-within path{fill:#242424}.lc:focus{fill:rgba(8, 8, 8, 1)}.mh:focus svg{color:#000000}.oh:focus{transform:scale(1.01)}.lh:active{border-style:none}</style><style type="text/css" data-fela-rehydration="546" data-fela-type="RULE" media="all and (min-width: 1080px)">.d{display:none}.bw{width:64px}.cg{margin:0 64px}.cv{height:48px}.dc{margin-bottom:52px}.do{margin-bottom:48px}.ef{font-size:14px}.eg{line-height:20px}.em{font-size:13px}.eo{padding:5px 12px}.fh{display:flex}.fy{margin-bottom:68px}.gc{max-width:680px}.hq{font-size:42px}.hr{margin-top:1.19em}.hs{margin-bottom:32px}.ht{line-height:52px}.hu{letter-spacing:-0.011em}.hz{align-items:center}.kb{border-top:solid 1px #F2F2F2}.kc{border-bottom:solid 1px #F2F2F2}.kd{margin:32px 0 0}.ke{padding:3px 8px}.kn> *{margin-right:24px}.ko> :last-child{margin-right:0}.lo{margin-top:0px}.mc{margin:0}.np{font-size:20px}.nq{margin-top:2.14em}.nr{line-height:32px}.ns{letter-spacing:-0.003em}.oc{margin-top:56px}.pc{font-size:24px}.pd{margin-top:1.95em}.pe{line-height:30px}.pf{letter-spacing:-0.016em}.pl{margin-top:0.94em}.pu{margin-top:1.14em}.qg{margin-top:1.72em}.qh{line-height:24px}.qi{letter-spacing:0}.ri{display:inline-block}.ro{margin-bottom:104px}.rs{flex-direction:row}.rv{margin-bottom:0}.rw{margin-right:20px}.sl{max-width:500px}.tn{margin-bottom:72px}.tv{padding-top:72px}</style><style type="text/css" data-fela-rehydration="546" data-fela-type="RULE" media="all and (max-width: 1079.98px)">.e{display:none}.ln{margin-top:0px}.rh{display:inline-block}</style><style type="text/css" data-fela-rehydration="546" data-fela-type="RULE" media="all and (max-width: 903.98px)">.f{display:none}.lm{margin-top:0px}.rg{display:inline-block}</style><style type="text/css" data-fela-rehydration="546" data-fela-type="RULE" media="all and (max-width: 727.98px)">.g{display:none}.lk{margin-top:0px}.ll{margin-right:0px}.rf{display:inline-block}</style><style type="text/css" data-fela-rehydration="546" data-fela-type="RULE" media="all and (max-width: 551.98px)">.h{display:none}.s{display:flex}.t{justify-content:space-between}.bs{width:24px}.cc{margin:0 24px}.cr{height:40px}.cy{margin-bottom:44px}.dk{margin-bottom:32px}.dx{font-size:13px}.dy{line-height:20px}.eh{padding:0px 8px 1px}.fu{margin-bottom:4px}.gw{font-size:32px}.gx{margin-top:1.01em}.gy{margin-bottom:24px}.gz{line-height:38px}.ha{letter-spacing:-0.014em}.hv{align-items:flex-start}.iy{flex-direction:column}.jb{margin-bottom:2px}.jp{margin:24px -24px 0}.jq{padding:0}.kf> *{margin-right:8px}.kg> :last-child{margin-right:24px}.kx{margin-left:0px}.li{margin-top:0px}.lj{margin-right:0px}.ly{margin:0}.mi{border:1px solid #F2F2F2}.mj{border-radius:99em}.mk{padding:0px 16px 0px 12px}.ml{height:38px}.mm{align-items:center}.mo svg{margin-right:8px}.mz{font-size:18px}.na{margin-top:1.56em}.nb{line-height:28px}.nc{letter-spacing:-0.003em}.ny{margin-top:40px}.om{font-size:20px}.on{margin-top:1.2em}.oo{line-height:24px}.op{letter-spacing:0}.ph{margin-top:0.67em}.pq{margin-top:1.34em}.pw{font-size:16px}.px{margin-top:1.23em}.re{display:inline-block}.rk{margin-bottom:96px}.sd{margin-bottom:20px}.se{margin-right:0}.sp{max-width:100%}.ss{font-size:24px}.st{line-height:30px}.su{letter-spacing:-0.016em}.tj{margin-bottom:64px}.tr{padding-top:48px}.mn:hover{border-color:#E5E5E5}</style><style type="text/css" data-fela-rehydration="546" data-fela-type="RULE" media="all and (min-width: 904px) and (max-width: 1079.98px)">.i{display:none}.bv{width:64px}.cf{margin:0 64px}.cu{height:48px}.db{margin-bottom:52px}.dn{margin-bottom:48px}.ed{font-size:14px}.ee{line-height:20px}.ek{font-size:13px}.el{padding:5px 12px}.fg{display:flex}.fx{margin-bottom:68px}.gb{max-width:680px}.hl{font-size:42px}.hm{margin-top:1.19em}.hn{margin-bottom:32px}.ho{line-height:52px}.hp{letter-spacing:-0.011em}.hy{align-items:center}.jx{border-top:solid 1px #F2F2F2}.jy{border-bottom:solid 1px #F2F2F2}.jz{margin:32px 0 0}.ka{padding:3px 8px}.kl> *{margin-right:24px}.km> :last-child{margin-right:0}.mb{margin:0}.nl{font-size:20px}.nm{margin-top:2.14em}.nn{line-height:32px}.no{letter-spacing:-0.003em}.ob{margin-top:56px}.oy{font-size:24px}.oz{margin-top:1.95em}.pa{line-height:30px}.pb{letter-spacing:-0.016em}.pk{margin-top:0.94em}.pt{margin-top:1.14em}.qd{margin-top:1.72em}.qe{line-height:24px}.qf{letter-spacing:0}.rn{margin-bottom:104px}.rt{flex-direction:row}.rx{margin-bottom:0}.ry{margin-right:20px}.sm{max-width:500px}.tm{margin-bottom:72px}.tu{padding-top:72px}</style><style type="text/css" data-fela-rehydration="546" data-fela-type="RULE" media="all and (min-width: 728px) and (max-width: 903.98px)">.j{display:none}.w{display:flex}.x{justify-content:space-between}.bu{width:64px}.ce{margin:0 48px}.ct{height:48px}.da{margin-bottom:52px}.dm{margin-bottom:48px}.eb{font-size:13px}.ec{line-height:20px}.ej{padding:0px 8px 1px}.fw{margin-bottom:68px}.ga{max-width:680px}.hg{font-size:42px}.hh{margin-top:1.19em}.hi{margin-bottom:32px}.hj{line-height:52px}.hk{letter-spacing:-0.011em}.hx{align-items:center}.jt{border-top:solid 1px #F2F2F2}.ju{border-bottom:solid 1px #F2F2F2}.jv{margin:32px 0 0}.jw{padding:3px 8px}.kj> *{margin-right:24px}.kk> :last-child{margin-right:0}.ma{margin:0}.nh{font-size:20px}.ni{margin-top:2.14em}.nj{line-height:32px}.nk{letter-spacing:-0.003em}.oa{margin-top:56px}.ou{font-size:24px}.ov{margin-top:1.95em}.ow{line-height:30px}.ox{letter-spacing:-0.016em}.pj{margin-top:0.94em}.ps{margin-top:1.14em}.qa{margin-top:1.72em}.qb{line-height:24px}.qc{letter-spacing:0}.rm{margin-bottom:104px}.ru{flex-direction:row}.rz{margin-bottom:0}.sa{margin-right:20px}.sn{max-width:500px}.tl{margin-bottom:72px}.tt{padding-top:72px}</style><style type="text/css" data-fela-rehydration="546" data-fela-type="RULE" media="all and (min-width: 552px) and (max-width: 727.98px)">.k{display:none}.u{display:flex}.v{justify-content:space-between}.bt{width:24px}.cd{margin:0 24px}.cs{height:40px}.cz{margin-bottom:44px}.dl{margin-bottom:32px}.dz{font-size:13px}.ea{line-height:20px}.ei{padding:0px 8px 1px}.fv{margin-bottom:4px}.hb{font-size:32px}.hc{margin-top:1.01em}.hd{margin-bottom:24px}.he{line-height:38px}.hf{letter-spacing:-0.014em}.hw{align-items:flex-start}.iz{flex-direction:column}.jc{margin-bottom:2px}.jr{margin:24px 0 0}.js{padding:0}.kh> *{margin-right:8px}.ki> :last-child{margin-right:8px}.ky{margin-left:0px}.lz{margin:0}.mp{border:1px solid #F2F2F2}.mq{border-radius:99em}.mr{padding:0px 16px 0px 12px}.ms{height:38px}.mt{align-items:center}.mv svg{margin-right:8px}.nd{font-size:18px}.ne{margin-top:1.56em}.nf{line-height:28px}.ng{letter-spacing:-0.003em}.nz{margin-top:40px}.oq{font-size:20px}.or{margin-top:1.2em}.os{line-height:24px}.ot{letter-spacing:0}.pi{margin-top:0.67em}.pr{margin-top:1.34em}.py{font-size:16px}.pz{margin-top:1.23em}.rl{margin-bottom:96px}.sb{margin-bottom:20px}.sc{margin-right:0}.so{max-width:100%}.sv{font-size:24px}.sw{line-height:30px}.sx{letter-spacing:-0.016em}.tk{margin-bottom:64px}.ts{padding-top:48px}.mu:hover{border-color:#E5E5E5}</style><style type="text/css" data-fela-rehydration="546" data-fela-type="RULE" media="print">.rc{display:none}</style><style type="text/css" data-fela-rehydration="546" data-fela-type="RULE" media="(orientation: landscape) and (max-width: 903.98px)">.jm{max-height:none}</style><style type="text/css" data-fela-rehydration="546" data-fela-type="RULE" media="(prefers-reduced-motion: no-preference)">.oe{transition:transform 300ms cubic-bezier(0.2, 0, 0.2, 1)}</style></head><body><div id="root"><div class="a b c"><div class="d e f g h i j k"></div><script>document.domain = document.domain;</script><div class="l c"><div class="l m n o c"><div class="p q r s t u v w x i d y z"><a class="du ag dv bf ak b am an ao ap aq ar as at s u w i d q dw z" href="https://rsci.app.link/?%24canonical_url=https%3A%2F%2Fmedium.com%2Fp%2Fc5fa1610d786&%7Efeature=LoOpenInAppButton&%7Echannel=ShowPostUnderCollection&source=---top_nav_layout_nav----------------------------------" rel="noopener follow">Open in app<svg xmlns="http://www.w3.org/2000/svg" width="10" height="10" fill="none" viewBox="0 0 10 10" class="dt"><path fill="currentColor" d="M.985 8.485a.375.375 0 1 0 .53.53zM8.75 1.25h.375A.375.375 0 0 0 8.75.875zM8.375 6.5a.375.375 0 1 0 .75 0zM3.5.875a.375.375 0 1 0 0 .75zm-1.985 8.14 7.5-7.5-.53-.53-7.5 7.5zm6.86-7.765V6.5h.75V1.25zM3.5 1.625h5.25v-.75H3.5z"></path></svg></a><div class="ab q"><p class="bf b dx dy dz ea eb ec ed ee ef eg du"><span><button class="bf b dx dy eh dz ea ei eb ec ej ek ee el em eg eo ep eq er es et eu ev ew ex ey ez fa fb fc fd bm fe ff" data-testid="headerSignUpButton">Sign up</button></span></p><div class="ax l"><p class="bf b dx dy dz ea eb ec ed ee ef eg du"><span><a class="af ag ah ai aj ak al am an ao ap aq ar as at" data-testid="headerSignInButton" rel="noopener follow" href="/m/signin?operation=login&redirect=https%3A%2F%2Fmedium.com%2Foutfit7%2Fthe-billion-events-infrastructure-c5fa1610d786&source=post_page---top_nav_layout_nav-----------------------global_nav-----------">Sign in</a></span></p></div></div></div><div class="p q r ab ac"><div class="ab q ae"><a class="af ag ah ai aj ak al am an ao ap aq ar as at ab" aria-label="Homepage" data-testid="headerMediumLogo" rel="noopener follow" href="/?source=---top_nav_layout_nav----------------------------------"><svg xmlns="http://www.w3.org/2000/svg" width="719" height="160" fill="none" viewBox="0 0 719 160" class="au av aw"><path fill="#242424" d="m174.104 9.734.215-.047V8.02H130.39L89.6 103.89 48.81 8.021H1.472v1.666l.212.047c8.018 1.81 12.09 4.509 12.09 14.242V137.93c0 9.734-4.087 12.433-12.106 14.243l-.212.047v1.671h32.118v-1.665l-.213-.048c-8.018-1.809-12.089-4.509-12.089-14.242V30.586l52.399 123.305h2.972l53.925-126.743V140.75c-.687 7.688-4.721 10.062-11.982 11.701l-.215.05v1.652h55.948v-1.652l-.215-.05c-7.269-1.639-11.4-4.013-12.087-11.701l-.037-116.774h.037c0-9.733 4.071-12.432 12.087-14.242m25.555 75.488c.915-20.474 8.268-35.252 20.606-35.507 3.806.063 6.998 1.312 9.479 3.714 5.272 5.118 7.751 15.812 7.368 31.793zm-.553 5.77h65.573v-.275c-.186-15.656-4.721-27.834-13.466-36.196-7.559-7.227-18.751-11.203-30.507-11.203h-.263c-6.101 0-13.584 1.48-18.909 4.16-6.061 2.807-11.407 7.003-15.855 12.511-7.161 8.874-11.499 20.866-12.554 34.343q-.05.606-.092 1.212a50 50 0 0 0-.065 1.151 85.807 85.807 0 0 0-.094 5.689c.71 30.524 17.198 54.917 46.483 54.917 25.705 0 40.675-18.791 44.407-44.013l-1.886-.664c-6.557 13.556-18.334 21.771-31.738 20.769-18.297-1.369-32.314-19.922-31.042-42.395m139.722 41.359c-2.151 5.101-6.639 7.908-12.653 7.908s-11.513-4.129-15.418-11.63c-4.197-8.053-6.405-19.436-6.405-32.92 0-28.067 8.729-46.22 22.24-46.22 5.657 0 10.111 2.807 12.236 7.704zm43.499 20.008c-8.019-1.897-12.089-4.722-12.089-14.951V1.309l-48.716 14.353v1.757l.299-.024c6.72-.543 11.278.386 13.925 2.83 2.072 1.915 3.082 4.853 3.082 8.987v18.66c-4.803-3.067-10.516-4.56-17.448-4.56-14.059 0-26.909 5.92-36.176 16.672-9.66 11.205-14.767 26.518-14.767 44.278-.003 31.72 15.612 53.039 38.851 53.039 13.595 0 24.533-7.449 29.54-20.013v16.865h43.711v-1.746zM424.1 19.819c0-9.904-7.468-17.374-17.375-17.374-9.859 0-17.573 7.632-17.573 17.374s7.721 17.374 17.573 17.374c9.907 0 17.375-7.47 17.375-17.374m11.499 132.546c-8.019-1.897-12.089-4.722-12.089-14.951h-.035V43.635l-43.714 12.551v1.705l.263.024c9.458.842 12.047 4.1 12.047 15.152v81.086h43.751v-1.746zm112.013 0c-8.018-1.897-12.089-4.722-12.089-14.951V43.635l-41.621 12.137v1.71l.246.026c7.733.813 9.967 4.257 9.967 15.36v59.279c-2.578 5.102-7.415 8.131-13.274 8.336-9.503 0-14.736-6.419-14.736-18.073V43.638l-43.714 12.55v1.703l.262.024c9.459.84 12.05 4.097 12.05 15.152v50.17a56.3 56.3 0 0 0 .91 10.444l.787 3.423c3.701 13.262 13.398 20.197 28.59 20.197 12.868 0 24.147-7.966 29.115-20.43v17.311h43.714v-1.747zm169.818 1.788v-1.749l-.213-.05c-8.7-2.006-12.089-5.789-12.089-13.49v-63.79c0-19.89-11.171-31.761-29.883-31.761-13.64 0-25.141 7.882-29.569 20.16-3.517-13.01-13.639-20.16-28.606-20.16-13.146 0-23.449 6.938-27.869 18.657V43.643L545.487 55.68v1.715l.263.024c9.345.829 12.047 4.181 12.047 14.95v81.784h40.787v-1.746l-.215-.053c-6.941-1.631-9.181-4.606-9.181-12.239V66.998c1.836-4.289 5.537-9.37 12.853-9.37 9.086 0 13.692 6.296 13.692 18.697v77.828h40.797v-1.746l-.215-.053c-6.94-1.631-9.18-4.606-9.18-12.239V75.066a42 42 0 0 0-.578-7.26c1.947-4.661 5.86-10.177 13.475-10.177 9.214 0 13.691 6.114 13.691 18.696v77.828z"></path></svg></a><div class="ax h"><div class="ab ay az ba bb q bc bd"><div class="bm" aria-hidden="false" aria-describedby="searchResults" aria-labelledby="searchResults"></div><div class="bn bo ab"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24"><path fill="currentColor" fill-rule="evenodd" d="M4.092 11.06a6.95 6.95 0 1 1 13.9 0 6.95 6.95 0 0 1-13.9 0m6.95-8.05a8.05 8.05 0 1 0 5.13 14.26l3.75 3.75a.56.56 0 1 0 .79-.79l-3.73-3.73A8.05 8.05 0 0 0 11.042 3z" clip-rule="evenodd"></path></svg></div><input role="combobox" aria-controls="searchResults" aria-expanded="false" aria-label="search" data-testid="headerSearchInput" tabindex="0" class="ay be bf bg z bh bi bj bk bl" placeholder="Search" value=""/></div></div></div><div class="h k w fg fh"><div class="fi ab"><span><a class="af ag ah ai aj ak al am an ao ap aq ar as at" data-testid="headerWriteButton" rel="noopener follow" href="/m/signin?operation=register&redirect=https%3A%2F%2Fmedium.com%2Fnew-story&source=---top_nav_layout_nav-----------------------new_post_topnav-----------"><div class="bf b bg z du fj fk ab q fl fm"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" aria-label="Write"><path fill="currentColor" d="M14 4a.5.5 0 0 0 0-1zm7 6a.5.5 0 0 0-1 0zm-7-7H4v1h10zM3 4v16h1V4zm1 17h16v-1H4zm17-1V10h-1v10zm-1 1a1 1 0 0 0 1-1h-1zM3 20a1 1 0 0 0 1 1v-1zM4 3a1 1 0 0 0-1 1h1z"></path><path stroke="currentColor" d="m17.5 4.5-8.458 8.458a.25.25 0 0 0-.06.098l-.824 2.47a.25.25 0 0 0 .316.316l2.47-.823a.25.25 0 0 0 .098-.06L19.5 6.5m-2-2 2.323-2.323a.25.25 0 0 1 .354 0l1.646 1.646a.25.25 0 0 1 0 .354L19.5 6.5m-2-2 2 2"></path></svg><div class="dt l">Write</div></div></a></span></div></div><div class="k j i d"><div class="fi ab"><a class="af ag ah ai aj ak al am an ao ap aq ar as at" data-testid="headerSearchButton" rel="noopener follow" href="/search?source=---top_nav_layout_nav----------------------------------"><div class="bf b bg z du fj fk ab q fl fm"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24" aria-label="Search"><path fill="currentColor" fill-rule="evenodd" d="M4.092 11.06a6.95 6.95 0 1 1 13.9 0 6.95 6.95 0 0 1-13.9 0m6.95-8.05a8.05 8.05 0 1 0 5.13 14.26l3.75 3.75a.56.56 0 1 0 .79-.79l-3.73-3.73A8.05 8.05 0 0 0 11.042 3z" clip-rule="evenodd"></path></svg></div></a></div></div><div class="fi h k j"><div class="ab q"><p class="bf b dx dy dz ea eb ec ed ee ef eg du"><span><button class="bf b dx dy eh dz ea ei eb ec ej ek ee el em eg eo ep eq er es et eu ev ew ex ey ez fa fb fc fd bm fe ff" data-testid="headerSignUpButton">Sign up</button></span></p><div class="ax l"><p class="bf b dx dy dz ea eb ec ed ee ef eg du"><span><a class="af ag ah ai aj ak al am an ao ap aq ar as at" data-testid="headerSignInButton" rel="noopener follow" href="/m/signin?operation=login&redirect=https%3A%2F%2Fmedium.com%2Foutfit7%2Fthe-billion-events-infrastructure-c5fa1610d786&source=post_page---top_nav_layout_nav-----------------------global_nav-----------">Sign in</a></span></p></div></div></div><div class="l" aria-hidden="false"><button class="ay fn am ab q ao fo fp fq" aria-label="user options menu" data-testid="headerUserIcon"><div class="l fj"><img alt="" class="l fd by bz ca cx" src="https://miro.medium.com/v2/resize:fill:64:64/1*dmbNkD5D-u45r44go_cf0g.png" width="32" height="32" loading="lazy" role="presentation"/><div class="fr by l bz ca fs n ay ft"></div></div></button></div></div></div><div class="l"><div class="fu fv fw fx fy l"><div class="ab cb"><div class="ci bh fz ga gb gc"></div></div><article><div class="l"><div class="l"><span class="l"></span><section><div><div class="fs gi gj gk gl gm"></div><div class="gn go gp gq gr"><div class="ab cb"><div class="ci bh fz ga gb gc"><div><h1 id="02c9" class="pw-post-title gs gt gu bf gv gw gx gy gz ha hb hc hd he hf hg hh hi hj hk hl hm hn ho hp hq hr hs ht hu bk" data-testid="storyTitle">The Billion Events Infrastructure</h1><div><div class="speechify-ignore ab cp"><div class="speechify-ignore bh l"><div class="hv hw hx hy hz ab"><div><div class="ab ia"><div><div class="bm" aria-hidden="false"><a rel="noopener follow" href="/@tilen.kavcic_63182?source=post_page---byline--c5fa1610d786--------------------------------"><div class="l ib ic by id ie"><div class="l fj"><img alt="Tilen Kavcic" class="l fd by dd de cx" src="https://miro.medium.com/v2/da:true/resize:fill:88:88/0*huBuE1NwJmzOfK05" width="44" height="44" loading="lazy" data-testid="authorPhoto"/><div class="if by l dd de fs n ig ft"></div></div></div></a></div></div><div class="ih ab fj"><div><div class="bm" aria-hidden="false"><a href="https://medium.com/outfit7?source=post_page---byline--c5fa1610d786--------------------------------" rel="noopener follow"><div class="l ii ij by id ik"><div class="l fj"><img alt="Outfit7" class="l fd by br il cx" src="https://miro.medium.com/v2/resize:fill:48:48/1*dCIQWfMya9g57UUL2ZzrJQ.png" width="24" height="24" loading="lazy" data-testid="publicationPhoto"/><div class="if by l br il fs n ig ft"></div></div></div></a></div></div></div></div></div><div class="bn bh l"><div class="ab"><div style="flex:1"><span class="bf b bg z bk"><div class="im ab q"><div class="ab q in"><div class="ab q"><div><div class="bm" aria-hidden="false"><p class="bf b io ip bk"><a class="af ag ah ai aj ak al am an ao ap aq ar iq" data-testid="authorName" rel="noopener follow" href="/@tilen.kavcic_63182?source=post_page---byline--c5fa1610d786--------------------------------">Tilen Kavcic</a></p></div></div></div><span class="ir is" aria-hidden="true"><span class="bf b bg z du">·</span></span><p class="bf b io ip du"><span><a class="it iu ah ai aj ak al am an ao ap aq ar ex iv iw" rel="noopener follow" href="/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fsubscribe%2Fuser%2Fc7d7326118b3&operation=register&redirect=https%3A%2F%2Fmedium.com%2Foutfit7%2Fthe-billion-events-infrastructure-c5fa1610d786&user=Tilen+Kavcic&userId=c7d7326118b3&source=post_page-c7d7326118b3--byline--c5fa1610d786---------------------post_header-----------">Follow</a></span></p></div></div></span></div></div><div class="l ix"><span class="bf b bg z du"><div class="ab cn iy iz ja"><div class="jb jc ab"><div class="bf b bg z du ab jd"><span class="je l ix">Published in</span><div><div class="l" aria-hidden="false"><a class="af ag ah ai aj ak al am an ao ap aq ar iq ab q" data-testid="publicationName" href="https://medium.com/outfit7?source=post_page---byline--c5fa1610d786--------------------------------" rel="noopener follow"><p class="bf b bg z jf jg jh ji jj jk jl jm bk">Outfit7</p></a></div></div></div><div class="h k"><span class="ir is" aria-hidden="true"><span class="bf b bg z du">·</span></span></div></div><span class="bf b bg z du"><div class="ab ae"><span data-testid="storyReadTime">5 min read</span><div class="jn jo l" aria-hidden="true"><span class="l" aria-hidden="true"><span class="bf b bg z du">·</span></span></div><span data-testid="storyPublishDate">Dec 13, 2022</span></div></span></div></span></div></div></div><div class="ab cp jp jq jr js jt ju jv jw jx jy jz ka kb kc kd ke"><div class="h k w fg fh q"><div class="ku l"><div class="ab q kv kw"><div class="pw-multi-vote-icon fj je kx ky kz"><span><a class="af ag ah ai aj ak al am an ao ap aq ar as at" data-testid="headerClapButton" rel="noopener follow" href="/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fvote%2Foutfit7%2Fc5fa1610d786&operation=register&redirect=https%3A%2F%2Fmedium.com%2Foutfit7%2Fthe-billion-events-infrastructure-c5fa1610d786&user=Tilen+Kavcic&userId=c7d7326118b3&source=---header_actions--c5fa1610d786---------------------clap_footer-----------"><div><div class="bm" aria-hidden="false"><div class="la ao lb lc ld le am lf lg lh kz"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" aria-label="clap"><path fill-rule="evenodd" d="M11.37.828 12 3.282l.63-2.454zM13.916 3.953l1.523-2.112-1.184-.39zM8.589 1.84l1.522 2.112-.337-2.501zM18.523 18.92c-.86.86-1.75 1.246-2.62 1.33a6 6 0 0 0 .407-.372c2.388-2.389 2.86-4.951 1.399-7.623l-.912-1.603-.79-1.672c-.26-.56-.194-.98.203-1.288a.7.7 0 0 1 .546-.132c.283.046.546.231.728.5l2.363 4.157c.976 1.624 1.141 4.237-1.324 6.702m-10.999-.438L3.37 14.328a.828.828 0 0 1 .585-1.408.83.83 0 0 1 .585.242l2.158 2.157a.365.365 0 0 0 .516-.516l-2.157-2.158-1.449-1.449a.826.826 0 0 1 1.167-1.17l3.438 3.44a.363.363 0 0 0 .516 0 .364.364 0 0 0 0-.516L5.293 9.513l-.97-.97a.826.826 0 0 1 0-1.166.84.84 0 0 1 1.167 0l.97.968 3.437 3.436a.36.36 0 0 0 .517 0 .366.366 0 0 0 0-.516L6.977 7.83a.82.82 0 0 1-.241-.584.82.82 0 0 1 .824-.826c.219 0 .43.087.584.242l5.787 5.787a.366.366 0 0 0 .587-.415l-1.117-2.363c-.26-.56-.194-.98.204-1.289a.7.7 0 0 1 .546-.132c.283.046.545.232.727.501l2.193 3.86c1.302 2.38.883 4.59-1.277 6.75-1.156 1.156-2.602 1.627-4.19 1.367-1.418-.236-2.866-1.033-4.079-2.246M10.75 5.971l2.12 2.12c-.41.502-.465 1.17-.128 1.89l.22.465-3.523-3.523a.8.8 0 0 1-.097-.368c0-.22.086-.428.241-.584a.847.847 0 0 1 1.167 0m7.355 1.705c-.31-.461-.746-.758-1.23-.837a1.44 1.44 0 0 0-1.11.275c-.312.24-.505.543-.59.881a1.74 1.74 0 0 0-.906-.465 1.47 1.47 0 0 0-.82.106l-2.182-2.182a1.56 1.56 0 0 0-2.2 0 1.54 1.54 0 0 0-.396.701 1.56 1.56 0 0 0-2.21-.01 1.55 1.55 0 0 0-.416.753c-.624-.624-1.649-.624-2.237-.037a1.557 1.557 0 0 0 0 2.2c-.239.1-.501.238-.715.453a1.56 1.56 0 0 0 0 2.2l.516.515a1.556 1.556 0 0 0-.753 2.615L7.01 19c1.32 1.319 2.909 2.189 4.475 2.449q.482.08.971.08c.85 0 1.653-.198 2.393-.579.231.033.46.054.686.054 1.266 0 2.457-.52 3.505-1.567 2.763-2.763 2.552-5.734 1.439-7.586z" clip-rule="evenodd"></path></svg></div></div></div></a></span></div><div class="pw-multi-vote-count l li lj lk ll lm ln lo"><p class="bf b dv z du"><span class="lp">--</span></p></div></div></div><div><div class="bm" aria-hidden="false"><button class="ao la lq lr ab q fk ls lt" aria-label="responses"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" class="lu"><path d="M18.006 16.803c1.533-1.456 2.234-3.325 2.234-5.321C20.24 7.357 16.709 4 12.191 4S4 7.357 4 11.482c0 4.126 3.674 7.482 8.191 7.482.817 0 1.622-.111 2.393-.327.231.2.48.391.744.559 1.06.693 2.203 1.044 3.399 1.044.224-.008.4-.112.486-.287a.49.49 0 0 0-.042-.518c-.495-.67-.845-1.364-1.04-2.057a4 4 0 0 1-.125-.598zm-3.122 1.055-.067-.223-.315.096a8 8 0 0 1-2.311.338c-4.023 0-7.292-2.955-7.292-6.587 0-3.633 3.269-6.588 7.292-6.588 4.014 0 7.112 2.958 7.112 6.593 0 1.794-.608 3.469-2.027 4.72l-.195.168v.255c0 .056 0 .151.016.295.025.231.081.478.154.733.154.558.398 1.117.722 1.659a5.3 5.3 0 0 1-2.165-.845c-.276-.176-.714-.383-.941-.59z"></path></svg></button></div></div></div><div class="ab q kf kg kh ki kj kk kl km kn ko kp kq kr ks kt"><div class="lv k j i d"></div><div class="h k"><div><div class="bm" aria-hidden="false"><span><a class="af ag ah ai aj ak al am an ao ap aq ar as at" data-testid="headerBookmarkButton" rel="noopener follow" href="/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fbookmark%2Fp%2Fc5fa1610d786&operation=register&redirect=https%3A%2F%2Fmedium.com%2Foutfit7%2Fthe-billion-events-infrastructure-c5fa1610d786&source=---header_actions--c5fa1610d786---------------------bookmark_footer-----------"><svg xmlns="http://www.w3.org/2000/svg" width="25" height="25" fill="none" viewBox="0 0 25 25" class="du lw" aria-label="Add to list bookmark button"><path fill="currentColor" d="M18 2.5a.5.5 0 0 1 1 0V5h2.5a.5.5 0 0 1 0 1H19v2.5a.5.5 0 1 1-1 0V6h-2.5a.5.5 0 0 1 0-1H18zM7 7a1 1 0 0 1 1-1h3.5a.5.5 0 0 0 0-1H8a2 2 0 0 0-2 2v14a.5.5 0 0 0 .805.396L12.5 17l5.695 4.396A.5.5 0 0 0 19 21v-8.5a.5.5 0 0 0-1 0v7.485l-5.195-4.012a.5.5 0 0 0-.61 0L7 19.985z"></path></svg></a></span></div></div></div><div class="fd lx cn"><div class="l ae"><div class="ab cb"><div class="ly lz ma mb mc md ci bh"><div class="ab"><div class="bm bh" aria-hidden="false"><div><div class="bm" aria-hidden="false"><button aria-label="Listen" data-testid="audioPlayButton" class="af fk ah ai aj ak al me an ao ap ex mf mg lt mh mi mj mk ml s mm mn mo mp mq mr ms u mt mu mv"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24"><path fill="currentColor" fill-rule="evenodd" d="M3 12a9 9 0 1 1 18 0 9 9 0 0 1-18 0m9-10C6.477 2 2 6.477 2 12s4.477 10 10 10 10-4.477 10-10S17.523 2 12 2m3.376 10.416-4.599 3.066a.5.5 0 0 1-.777-.416V8.934a.5.5 0 0 1 .777-.416l4.599 3.066a.5.5 0 0 1 0 .832" clip-rule="evenodd"></path></svg><div class="j i d"><p class="bf b bg z du">Listen</p></div></button></div></div></div></div></div></div></div></div><div class="bm" aria-hidden="false" aria-describedby="postFooterSocialMenu" aria-labelledby="postFooterSocialMenu"><div><div class="bm" aria-hidden="false"><button aria-controls="postFooterSocialMenu" aria-expanded="false" aria-label="Share Post" data-testid="headerSocialShareButton" class="af fk ah ai aj ak al me an ao ap ex mf mg lt mh mi mj mk ml s mm mn mo mp mq mr ms u mt mu mv"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24"><path fill="currentColor" fill-rule="evenodd" d="M15.218 4.931a.4.4 0 0 1-.118.132l.012.006a.45.45 0 0 1-.292.074.5.5 0 0 1-.3-.13l-2.02-2.02v7.07c0 .28-.23.5-.5.5s-.5-.22-.5-.5v-7.04l-2 2a.45.45 0 0 1-.57.04h-.02a.4.4 0 0 1-.16-.3.4.4 0 0 1 .1-.32l2.8-2.8a.5.5 0 0 1 .7 0l2.8 2.79a.42.42 0 0 1 .068.498m-.106.138.008.004v-.01zM16 7.063h1.5a2 2 0 0 1 2 2v10a2 2 0 0 1-2 2h-11c-1.1 0-2-.9-2-2v-10a2 2 0 0 1 2-2H8a.5.5 0 0 1 .35.15.5.5 0 0 1 .15.35.5.5 0 0 1-.15.35.5.5 0 0 1-.35.15H6.4c-.5 0-.9.4-.9.9v10.2a.9.9 0 0 0 .9.9h11.2c.5 0 .9-.4.9-.9v-10.2c0-.5-.4-.9-.9-.9H16a.5.5 0 0 1 0-1" clip-rule="evenodd"></path></svg><div class="j i d"><p class="bf b bg z du">Share</p></div></button></div></div></div></div></div></div></div></div></div><p id="858a" class="pw-post-body-paragraph mw mx gu my b mz na nb nc nd ne nf ng nh ni nj nk nl nm nn no np nq nr ns nt gn bk"><strong class="my gv"><em class="nu">TL;DR: </em></strong><em class="nu">Outfit7 is, at its core, a data-driven company. Collecting and analyzing performance metrics from our games holds a lot of interesting challenges. In this article, I’ll explore our infrastructure design decisions and what we have learned along the way.</em></p><figure class="ny nz oa ob oc od nv nw paragraph-image"><div role="button" tabindex="0" class="oe of fj og bh oh"><div class="nv nw nx"><picture><source srcSet="https://miro.medium.com/v2/resize:fit:640/format:webp/0*YpS6May9aFFtbR0D 640w, https://miro.medium.com/v2/resize:fit:720/format:webp/0*YpS6May9aFFtbR0D 720w, https://miro.medium.com/v2/resize:fit:750/format:webp/0*YpS6May9aFFtbR0D 750w, https://miro.medium.com/v2/resize:fit:786/format:webp/0*YpS6May9aFFtbR0D 786w, https://miro.medium.com/v2/resize:fit:828/format:webp/0*YpS6May9aFFtbR0D 828w, https://miro.medium.com/v2/resize:fit:1100/format:webp/0*YpS6May9aFFtbR0D 1100w, https://miro.medium.com/v2/resize:fit:1400/format:webp/0*YpS6May9aFFtbR0D 1400w" sizes="(min-resolution: 4dppx) and (max-width: 700px) 50vw, (-webkit-min-device-pixel-ratio: 4) and (max-width: 700px) 50vw, (min-resolution: 3dppx) and (max-width: 700px) 67vw, (-webkit-min-device-pixel-ratio: 3) and (max-width: 700px) 65vw, (min-resolution: 2.5dppx) and (max-width: 700px) 80vw, (-webkit-min-device-pixel-ratio: 2.5) and (max-width: 700px) 80vw, (min-resolution: 2dppx) and (max-width: 700px) 100vw, (-webkit-min-device-pixel-ratio: 2) and (max-width: 700px) 100vw, 700px" type="image/webp"/><source data-testid="og" srcSet="https://miro.medium.com/v2/resize:fit:640/0*YpS6May9aFFtbR0D 640w, https://miro.medium.com/v2/resize:fit:720/0*YpS6May9aFFtbR0D 720w, https://miro.medium.com/v2/resize:fit:750/0*YpS6May9aFFtbR0D 750w, https://miro.medium.com/v2/resize:fit:786/0*YpS6May9aFFtbR0D 786w, https://miro.medium.com/v2/resize:fit:828/0*YpS6May9aFFtbR0D 828w, https://miro.medium.com/v2/resize:fit:1100/0*YpS6May9aFFtbR0D 1100w, https://miro.medium.com/v2/resize:fit:1400/0*YpS6May9aFFtbR0D 1400w" sizes="(min-resolution: 4dppx) and (max-width: 700px) 50vw, (-webkit-min-device-pixel-ratio: 4) and (max-width: 700px) 50vw, (min-resolution: 3dppx) and (max-width: 700px) 67vw, (-webkit-min-device-pixel-ratio: 3) and (max-width: 700px) 65vw, (min-resolution: 2.5dppx) and (max-width: 700px) 80vw, (-webkit-min-device-pixel-ratio: 2.5) and (max-width: 700px) 80vw, (min-resolution: 2dppx) and (max-width: 700px) 100vw, (-webkit-min-device-pixel-ratio: 2) and (max-width: 700px) 100vw, 700px"/><img alt="" class="bh md oi c" width="700" height="367" loading="eager" role="presentation"/></picture></div></div></figure><p id="3ee6" class="pw-post-body-paragraph mw mx gu my b mz na nb nc nd ne nf ng nh ni nj nk nl nm nn no np nq nr ns nt gn bk">Hi, my name is Tilen Kavčič. I’m one of the backend core members at Outfit7, specializing in cloud infrastructure and R&D. Our team’s mission is to create high performance, scalable solutions that support the growth of our company. I’m part software engineer, part data engineer, with an interest in DevOps. The best part of my job is getting the chance to create solutions that simplify development of new software. My current mission is to modernize our data stack, bringing software engineering best practices to our data teams.</p><h1 id="d5e6" class="oj ok gu bf ol om on oo op oq or os ot ou ov ow ox oy oz pa pb pc pd pe pf pg bk">Background</h1><p id="1dbe" class="pw-post-body-paragraph mw mx gu my b mz ph nb nc nd pi nf ng nh pj nj nk nl pk nn no np pl nr ns nt gn bk">At Outfit7, we believe that all our games should look, play and feel good. With 430 million monthly active users, it can be a challenge to get a clear picture of how our games and company are performing. The data we gather helps us understand how we perform in the fast-paced mobile gaming market. Getting fast feedback is key. Because of this, we need infrastructure that is scalable and reliable, and which, ideally, doesn’t hurt the company’s wallet too much. Over the past decade we’ve continuously improved our infrastructure to accommodate our growing user base. Today, we handle half a million row inserts per second — let me show you how. Welcome to the Billion Events Infrastructure.</p><h1 id="97b5" class="oj ok gu bf ol om on oo op oq or os ot ou ov ow ox oy oz pa pb pc pd pe pf pg bk">Infrastructure</h1><figure class="ny nz oa ob oc od nv nw paragraph-image"><div role="button" tabindex="0" class="oe of fj og bh oh"><div class="nv nw pm"><picture><source srcSet="https://miro.medium.com/v2/resize:fit:640/format:webp/0*Zek8d75_0P4uhYkI 640w, https://miro.medium.com/v2/resize:fit:720/format:webp/0*Zek8d75_0P4uhYkI 720w, https://miro.medium.com/v2/resize:fit:750/format:webp/0*Zek8d75_0P4uhYkI 750w, https://miro.medium.com/v2/resize:fit:786/format:webp/0*Zek8d75_0P4uhYkI 786w, https://miro.medium.com/v2/resize:fit:828/format:webp/0*Zek8d75_0P4uhYkI 828w, https://miro.medium.com/v2/resize:fit:1100/format:webp/0*Zek8d75_0P4uhYkI 1100w, https://miro.medium.com/v2/resize:fit:1400/format:webp/0*Zek8d75_0P4uhYkI 1400w" sizes="(min-resolution: 4dppx) and (max-width: 700px) 50vw, (-webkit-min-device-pixel-ratio: 4) and (max-width: 700px) 50vw, (min-resolution: 3dppx) and (max-width: 700px) 67vw, (-webkit-min-device-pixel-ratio: 3) and (max-width: 700px) 65vw, (min-resolution: 2.5dppx) and (max-width: 700px) 80vw, (-webkit-min-device-pixel-ratio: 2.5) and (max-width: 700px) 80vw, (min-resolution: 2dppx) and (max-width: 700px) 100vw, (-webkit-min-device-pixel-ratio: 2) and (max-width: 700px) 100vw, 700px" type="image/webp"/><source data-testid="og" srcSet="https://miro.medium.com/v2/resize:fit:640/0*Zek8d75_0P4uhYkI 640w, https://miro.medium.com/v2/resize:fit:720/0*Zek8d75_0P4uhYkI 720w, https://miro.medium.com/v2/resize:fit:750/0*Zek8d75_0P4uhYkI 750w, https://miro.medium.com/v2/resize:fit:786/0*Zek8d75_0P4uhYkI 786w, https://miro.medium.com/v2/resize:fit:828/0*Zek8d75_0P4uhYkI 828w, https://miro.medium.com/v2/resize:fit:1100/0*Zek8d75_0P4uhYkI 1100w, https://miro.medium.com/v2/resize:fit:1400/0*Zek8d75_0P4uhYkI 1400w" sizes="(min-resolution: 4dppx) and (max-width: 700px) 50vw, (-webkit-min-device-pixel-ratio: 4) and (max-width: 700px) 50vw, (min-resolution: 3dppx) and (max-width: 700px) 67vw, (-webkit-min-device-pixel-ratio: 3) and (max-width: 700px) 65vw, (min-resolution: 2.5dppx) and (max-width: 700px) 80vw, (-webkit-min-device-pixel-ratio: 2.5) and (max-width: 700px) 80vw, (min-resolution: 2dppx) and (max-width: 700px) 100vw, (-webkit-min-device-pixel-ratio: 2) and (max-width: 700px) 100vw, 700px"/><img alt="" class="bh md oi c" width="700" height="245" loading="lazy" role="presentation"/></picture></div></div></figure><p id="2381" class="pw-post-body-paragraph mw mx gu my b mz na nb nc nd ne nf ng nh ni nj nk nl nm nn no np nq nr ns nt gn bk">Let’s start with an overview of our infrastructure. We receive data from various data sources, like game usage data, third party services, etc., which we save in persistent storage. Here, we’ll focus only on data sent by our games, because this is what forms the backbone of our analyses. From data gathering endpoints to persistent storage, all is hosted on the Google Cloud Platform. Let’s start with the first layer, at our API endpoints:</p><ol class=""><li id="7759" class="mw mx gu my b mz na nb nc nd ne nf ng nh ni nj nk nl nm nn no np nq nr ns nt pn po pp bk"><strong class="my gv">API endpoints</strong><br/>We run our data-gathering endpoints on different managed services (App Engine, Cloud Run and Kubernetes Engine). Our main endpoint, where we gather usage data, runs on Kubernetes Engine. We chose Kubernetes because it offers a nice balance between performance and cost. Because we have a global player base, we run three clusters, each in different regions (US, EU and Asia). Our main cluster resides in the US, where we get the most traffic, while the Asia cluster serves as the endpoint for our users in China. We also set up one cluster in Europe which is our backfall endpoint in the event of endpoint failures in US and Asia clusters. All data is validated and pushed to our data delivery layer.</li><li id="bf36" class="mw mx gu my b mz pq nb nc nd pr nf ng nh ps nj nk nl pt nn no np pu nr ns nt pn po pp bk"><strong class="my gv">Data delivery</strong><br/>This second layer is a middleware between our API endpoints and data processing workloads. It provides asynchronous delivery of messages with the benefit of keeping our data safe until it’s written to persistent storage. We use Pub/Sub (similar to Apache Kafka) which provides a simple, reliable and scalable message delivery service.</li><li id="943f" class="mw mx gu my b mz pq nb nc nd pr nf ng nh ps nj nk nl pt nn no np pu nr ns nt pn po pp bk"><strong class="my gv">Data processing<br/></strong>In this layer, we do some light data transformation (unbatching the client data) and streaming data into persistent storage. Our main framework to do this is Apache Beam which runs on managed cloud infrastructure called Dataflow. This is a tried and tested way to do ETL pipelines because it provides a reliable way of inserting data into persistent storage.</li><li id="dac0" class="mw mx gu my b mz pq nb nc nd pr nf ng nh ps nj nk nl pt nn no np pu nr ns nt pn po pp bk"><strong class="my gv">Data storage<br/></strong>After all this, the data is finally written into BigQuery. It’s our main data warehouse, which powers all of our analytics workload. It’s fast, reliable and just what we need to store and analyze our data.</li></ol><p id="8efe" class="pw-post-body-paragraph mw mx gu my b mz na nb nc nd ne nf ng nh ni nj nk nl nm nn no np nq nr ns nt gn bk">All these layers combined create a scalable and reliable infrastructure that can receive, process and ingest more than half a million rows of data per second, or five terabytes of new data per day. Our data teams query 60 petabytes per year with zero downtime and zero maintenance from the core engineering team, just by using BigQuery. By using a simple approach and managed services a single person can maintain this infrastructure. This enables us to focus on developing new solutions that add value to our company.</p><h1 id="e88e" class="oj ok gu bf ol om on oo op oq or os ot ou ov ow ox oy oz pa pb pc pd pe pf pg bk">What we learned</h1><h2 id="ef35" class="pv ok gu bf ol pw px dy op py pz ea ot nh qa qb qc nl qd qe qf np qg qh qi qj bk">Development environment</h2><p id="aa72" class="pw-post-body-paragraph mw mx gu my b mz ph nb nc nd pi nf ng nh pj nj nk nl pk nn no np pl nr ns nt gn bk">These environments are important because they create a safe space where engineers can create and test new features. Without them you are always one step away from breaking the production environment. Our team takes these environments seriously. At the very beginning, we separate production and development services by using access permissions and configuration files. Using infrastructure as code (we use Terraform) speeds up creating development and production environments. If you want to create a better development environment we recommend looking at The Twelve-Factor App [1].</p><h2 id="5c49" class="pv ok gu bf ol pw px dy op py pz ea ot nh qa qb qc nl qd qe qf np qg qh qi qj bk">Tests</h2><p id="e55a" class="pw-post-body-paragraph mw mx gu my b mz ph nb nc nd pi nf ng nh pj nj nk nl pk nn no np pl nr ns nt gn bk">Probably the most important part of any service is the tests. Unit and integration tests provide a good overview if your services are working as expected. And for core services that have a direct impact on the company, tests are a must-have. We had great success detecting bugs in our code before releasing our code into production. The key note here is that tests should be meaningful and should test the core functionality of our service. To build upon this we use CI/CD to automate the releasing and testing phases which cuts the manual labor immensely.</p><h2 id="b337" class="pv ok gu bf ol pw px dy op py pz ea ot nh qa qb qc nl qd qe qf np qg qh qi qj bk">Metrics</h2><p id="2fc1" class="pw-post-body-paragraph mw mx gu my b mz ph nb nc nd pi nf ng nh pj nj nk nl pk nn no np pl nr ns nt gn bk">While tests primarily cover our core business logic during development, we still need to monitor the behavior and health of our services during and after deployment. To do this, we create metrics that give us real-time insights into how our service performs. On top of that, we build alert policies that inform us when something is wrong. We also have automatic abort functionality in our CI/CD pipeline if the metrics are above a certain threshold. But creating helpful metrics can be tricky. You can either over-monitor and overload your on-call people with alerts or under-monitor and overlook issues that are bad for the company’s performance. I recommend reading Google’s SRE book, which will help you find the middle ground [2].</p><h2 id="b78a" class="pv ok gu bf ol pw px dy op py pz ea ot nh qa qb qc nl qd qe qf np qg qh qi qj bk">Simplicity</h2><p id="78aa" class="pw-post-body-paragraph mw mx gu my b mz ph nb nc nd pi nf ng nh pj nj nk nl pk nn no np pl nr ns nt gn bk">Adding new features is inevitable, even when the core infrastructure is intact. We need to keep in mind that making decisions that require complex systems will hurt your team in the long run. While designing your systems you need to balance complexity and costs. Usually managed cloud solutions like Pub/Sub are the best way to reduce complexity, but they can be more costly. Our team uses managed cloud solutions because they reduce the overall load on the engineers. We also simplify our release process. We have one rule, if you push it to main it will be tested and released. No more scripts and documents on how to release new code.</p><blockquote class="qk ql qm"><p id="f079" class="mw mx nu my b mz na nb nc nd ne nf ng nh ni nj nk nl nm nn no np nq nr ns nt gn bk">[1]: https://12factor.net/</p><p id="2b13" class="mw mx nu my b mz na nb nc nd ne nf ng nh ni nj nk nl nm nn no np nq nr ns nt gn bk">[2]: https://sre.google/workbook/monitoring/</p></blockquote><h1 id="9f33" class="oj ok gu bf ol om on oo op oq or os ot ou ov ow ox oy oz pa pb pc pd pe pf pg bk">Conclusion</h1><p id="e8b9" class="pw-post-body-paragraph mw mx gu my b mz ph nb nc nd pi nf ng nh pj nj nk nl pk nn no np pl nr ns nt gn bk">In this article we looked at our core data infrastructure, which is responsible for receiving and saving the analytical data from our games. Due to the large amount of data we receive, we needed to create a reliable infrastructure design that would scale with our growth. Choosing the correct technology is just one part of creating such a system. Any core system should be designed with simplicity in mind with tests, metrics, logging and development environments that give developers a safety net. We went through many iterations of our system over the last couple of years, each time learning something new.</p></div></div></div></div></section></div></div></article></div><div class="ab cb"><div class="ci bh fz ga gb gc"><div class="qn qo ab ja"><div class="qp ab"><a class="qq ay am ao" rel="noopener follow" href="/tag/backend?source=post_page-----c5fa1610d786--------------------------------"><div class="qr fj cx qs ge qt qu bf b bg z bk qv">Backend</div></a></div><div class="qp ab"><a class="qq ay am ao" rel="noopener follow" href="/tag/data-engineering?source=post_page-----c5fa1610d786--------------------------------"><div class="qr fj cx qs ge qt qu bf b bg z bk qv">Data Engineering</div></a></div><div class="qp ab"><a class="qq ay am ao" rel="noopener follow" href="/tag/google-cloud-platform?source=post_page-----c5fa1610d786--------------------------------"><div class="qr fj cx qs ge qt qu bf b bg z bk qv">Google Cloud Platform</div></a></div><div class="qp ab"><a class="qq ay am ao" rel="noopener follow" href="/tag/analytics?source=post_page-----c5fa1610d786--------------------------------"><div class="qr fj cx qs ge qt qu bf b bg z bk qv">Analytics</div></a></div><div class="qp ab"><a class="qq ay am ao" rel="noopener follow" href="/tag/programming?source=post_page-----c5fa1610d786--------------------------------"><div class="qr fj cx qs ge qt qu bf b bg z bk qv">Programming</div></a></div></div></div></div><div class="l"></div><footer class="qw qx qy qz ra ab q rb ik c"><div class="l ae"><div class="ab cb"><div class="ci bh fz ga gb gc"><div class="ab cp rc"><div class="ab q kv"><div class="rd l"><span class="l re rf rg e d"><div class="ab q kv kw"><div class="pw-multi-vote-icon fj je kx ky kz"><span><a class="af ag ah ai aj ak al am an ao ap aq ar as at" data-testid="footerClapButton" rel="noopener follow" href="/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fvote%2Foutfit7%2Fc5fa1610d786&operation=register&redirect=https%3A%2F%2Fmedium.com%2Foutfit7%2Fthe-billion-events-infrastructure-c5fa1610d786&user=Tilen+Kavcic&userId=c7d7326118b3&source=---footer_actions--c5fa1610d786---------------------clap_footer-----------"><div><div class="bm" aria-hidden="false"><div class="la ao lb lc ld le am lf lg lh kz"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" aria-label="clap"><path fill-rule="evenodd" d="M11.37.828 12 3.282l.63-2.454zM13.916 3.953l1.523-2.112-1.184-.39zM8.589 1.84l1.522 2.112-.337-2.501zM18.523 18.92c-.86.86-1.75 1.246-2.62 1.33a6 6 0 0 0 .407-.372c2.388-2.389 2.86-4.951 1.399-7.623l-.912-1.603-.79-1.672c-.26-.56-.194-.98.203-1.288a.7.7 0 0 1 .546-.132c.283.046.546.231.728.5l2.363 4.157c.976 1.624 1.141 4.237-1.324 6.702m-10.999-.438L3.37 14.328a.828.828 0 0 1 .585-1.408.83.83 0 0 1 .585.242l2.158 2.157a.365.365 0 0 0 .516-.516l-2.157-2.158-1.449-1.449a.826.826 0 0 1 1.167-1.17l3.438 3.44a.363.363 0 0 0 .516 0 .364.364 0 0 0 0-.516L5.293 9.513l-.97-.97a.826.826 0 0 1 0-1.166.84.84 0 0 1 1.167 0l.97.968 3.437 3.436a.36.36 0 0 0 .517 0 .366.366 0 0 0 0-.516L6.977 7.83a.82.82 0 0 1-.241-.584.82.82 0 0 1 .824-.826c.219 0 .43.087.584.242l5.787 5.787a.366.366 0 0 0 .587-.415l-1.117-2.363c-.26-.56-.194-.98.204-1.289a.7.7 0 0 1 .546-.132c.283.046.545.232.727.501l2.193 3.86c1.302 2.38.883 4.59-1.277 6.75-1.156 1.156-2.602 1.627-4.19 1.367-1.418-.236-2.866-1.033-4.079-2.246M10.75 5.971l2.12 2.12c-.41.502-.465 1.17-.128 1.89l.22.465-3.523-3.523a.8.8 0 0 1-.097-.368c0-.22.086-.428.241-.584a.847.847 0 0 1 1.167 0m7.355 1.705c-.31-.461-.746-.758-1.23-.837a1.44 1.44 0 0 0-1.11.275c-.312.24-.505.543-.59.881a1.74 1.74 0 0 0-.906-.465 1.47 1.47 0 0 0-.82.106l-2.182-2.182a1.56 1.56 0 0 0-2.2 0 1.54 1.54 0 0 0-.396.701 1.56 1.56 0 0 0-2.21-.01 1.55 1.55 0 0 0-.416.753c-.624-.624-1.649-.624-2.237-.037a1.557 1.557 0 0 0 0 2.2c-.239.1-.501.238-.715.453a1.56 1.56 0 0 0 0 2.2l.516.515a1.556 1.556 0 0 0-.753 2.615L7.01 19c1.32 1.319 2.909 2.189 4.475 2.449q.482.08.971.08c.85 0 1.653-.198 2.393-.579.231.033.46.054.686.054 1.266 0 2.457-.52 3.505-1.567 2.763-2.763 2.552-5.734 1.439-7.586z" clip-rule="evenodd"></path></svg></div></div></div></a></span></div><div class="pw-multi-vote-count l li lj lk ll lm ln lo"><p class="bf b dv z du"><span class="lp">--</span></p></div></div></span><span class="l h g f rh ri"><div class="ab q kv kw"><div class="pw-multi-vote-icon fj je kx ky kz"><span><a class="af ag ah ai aj ak al am an ao ap aq ar as at" data-testid="footerClapButton" rel="noopener follow" href="/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fvote%2Foutfit7%2Fc5fa1610d786&operation=register&redirect=https%3A%2F%2Fmedium.com%2Foutfit7%2Fthe-billion-events-infrastructure-c5fa1610d786&user=Tilen+Kavcic&userId=c7d7326118b3&source=---footer_actions--c5fa1610d786---------------------clap_footer-----------"><div><div class="bm" aria-hidden="false"><div class="la ao lb lc ld le am lf lg lh kz"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" aria-label="clap"><path fill-rule="evenodd" d="M11.37.828 12 3.282l.63-2.454zM13.916 3.953l1.523-2.112-1.184-.39zM8.589 1.84l1.522 2.112-.337-2.501zM18.523 18.92c-.86.86-1.75 1.246-2.62 1.33a6 6 0 0 0 .407-.372c2.388-2.389 2.86-4.951 1.399-7.623l-.912-1.603-.79-1.672c-.26-.56-.194-.98.203-1.288a.7.7 0 0 1 .546-.132c.283.046.546.231.728.5l2.363 4.157c.976 1.624 1.141 4.237-1.324 6.702m-10.999-.438L3.37 14.328a.828.828 0 0 1 .585-1.408.83.83 0 0 1 .585.242l2.158 2.157a.365.365 0 0 0 .516-.516l-2.157-2.158-1.449-1.449a.826.826 0 0 1 1.167-1.17l3.438 3.44a.363.363 0 0 0 .516 0 .364.364 0 0 0 0-.516L5.293 9.513l-.97-.97a.826.826 0 0 1 0-1.166.84.84 0 0 1 1.167 0l.97.968 3.437 3.436a.36.36 0 0 0 .517 0 .366.366 0 0 0 0-.516L6.977 7.83a.82.82 0 0 1-.241-.584.82.82 0 0 1 .824-.826c.219 0 .43.087.584.242l5.787 5.787a.366.366 0 0 0 .587-.415l-1.117-2.363c-.26-.56-.194-.98.204-1.289a.7.7 0 0 1 .546-.132c.283.046.545.232.727.501l2.193 3.86c1.302 2.38.883 4.59-1.277 6.75-1.156 1.156-2.602 1.627-4.19 1.367-1.418-.236-2.866-1.033-4.079-2.246M10.75 5.971l2.12 2.12c-.41.502-.465 1.17-.128 1.89l.22.465-3.523-3.523a.8.8 0 0 1-.097-.368c0-.22.086-.428.241-.584a.847.847 0 0 1 1.167 0m7.355 1.705c-.31-.461-.746-.758-1.23-.837a1.44 1.44 0 0 0-1.11.275c-.312.24-.505.543-.59.881a1.74 1.74 0 0 0-.906-.465 1.47 1.47 0 0 0-.82.106l-2.182-2.182a1.56 1.56 0 0 0-2.2 0 1.54 1.54 0 0 0-.396.701 1.56 1.56 0 0 0-2.21-.01 1.55 1.55 0 0 0-.416.753c-.624-.624-1.649-.624-2.237-.037a1.557 1.557 0 0 0 0 2.2c-.239.1-.501.238-.715.453a1.56 1.56 0 0 0 0 2.2l.516.515a1.556 1.556 0 0 0-.753 2.615L7.01 19c1.32 1.319 2.909 2.189 4.475 2.449q.482.08.971.08c.85 0 1.653-.198 2.393-.579.231.033.46.054.686.054 1.266 0 2.457-.52 3.505-1.567 2.763-2.763 2.552-5.734 1.439-7.586z" clip-rule="evenodd"></path></svg></div></div></div></a></span></div><div class="pw-multi-vote-count l li lj lk ll lm ln lo"><p class="bf b dv z du"><span class="lp">--</span></p></div></div></span></div><div class="bq ab"><div><div class="bm" aria-hidden="false"><button class="ao la lq lr ab q fk ls lt" aria-label="responses"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" viewBox="0 0 24 24" class="lu"><path d="M18.006 16.803c1.533-1.456 2.234-3.325 2.234-5.321C20.24 7.357 16.709 4 12.191 4S4 7.357 4 11.482c0 4.126 3.674 7.482 8.191 7.482.817 0 1.622-.111 2.393-.327.231.2.48.391.744.559 1.06.693 2.203 1.044 3.399 1.044.224-.008.4-.112.486-.287a.49.49 0 0 0-.042-.518c-.495-.67-.845-1.364-1.04-2.057a4 4 0 0 1-.125-.598zm-3.122 1.055-.067-.223-.315.096a8 8 0 0 1-2.311.338c-4.023 0-7.292-2.955-7.292-6.587 0-3.633 3.269-6.588 7.292-6.588 4.014 0 7.112 2.958 7.112 6.593 0 1.794-.608 3.469-2.027 4.72l-.195.168v.255c0 .056 0 .151.016.295.025.231.081.478.154.733.154.558.398 1.117.722 1.659a5.3 5.3 0 0 1-2.165-.845c-.276-.176-.714-.383-.941-.59z"></path></svg></button></div></div></div></div><div class="ab q"><div class="rj l ix"><div><div class="bm" aria-hidden="false"><span><a class="af ag ah ai aj ak al am an ao ap aq ar as at" data-testid="footerBookmarkButton" rel="noopener follow" href="/m/signin?actionUrl=https%3A%2F%2Fmedium.com%2F_%2Fbookmark%2Fp%2Fc5fa1610d786&operation=register&redirect=https%3A%2F%2Fmedium.com%2Foutfit7%2Fthe-billion-events-infrastructure-c5fa1610d786&source=---footer_actions--c5fa1610d786---------------------bookmark_footer-----------"><svg xmlns="http://www.w3.org/2000/svg" width="25" height="25" fill="none" viewBox="0 0 25 25" class="du lw" aria-label="Add to list bookmark button"><path fill="currentColor" d="M18 2.5a.5.5 0 0 1 1 0V5h2.5a.5.5 0 0 1 0 1H19v2.5a.5.5 0 1 1-1 0V6h-2.5a.5.5 0 0 1 0-1H18zM7 7a1 1 0 0 1 1-1h3.5a.5.5 0 0 0 0-1H8a2 2 0 0 0-2 2v14a.5.5 0 0 0 .805.396L12.5 17l5.695 4.396A.5.5 0 0 0 19 21v-8.5a.5.5 0 0 0-1 0v7.485l-5.195-4.012a.5.5 0 0 0-.61 0L7 19.985z"></path></svg></a></span></div></div></div><div class="rj l ix"><div class="bm" aria-hidden="false" aria-describedby="postFooterSocialMenu" aria-labelledby="postFooterSocialMenu"><div><div class="bm" aria-hidden="false"><button aria-controls="postFooterSocialMenu" aria-expanded="false" aria-label="Share Post" data-testid="footerSocialShareButton" class="af fk ah ai aj ak al me an ao ap ex mf mg lt mh"><svg xmlns="http://www.w3.org/2000/svg" width="24" height="24" fill="none" viewBox="0 0 24 24"><path fill="currentColor" fill-rule="evenodd" d="M15.218 4.931a.4.4 0 0 1-.118.132l.012.006a.45.45 0 0 1-.292.074.5.5 0 0 1-.3-.13l-2.02-2.02v7.07c0 .28-.23.5-.5.5s-.5-.22-.5-.5v-7.04l-2 2a.45.45 0 0 1-.57.04h-.02a.4.4 0 0 1-.16-.3.4.4 0 0 1 .1-.32l2.8-2.8a.5.5 0 0 1 .7 0l2.8 2.79a.42.42 0 0 1 .068.498m-.106.138.008.004v-.01zM16 7.063h1.5a2 2 0 0 1 2 2v10a2 2 0 0 1-2 2h-11c-1.1 0-2-.9-2-2v-10a2 2 0 0 1 2-2H8a.5.5 0 0 1 .35.15.5.5 0 0 1 .15.35.5.5 0 0 1-.15.35.5.5 0 0 1-.35.15H6.4c-.5 0-.9.4-.9.9v10.2a.9.9 0 0 0 .9.9h11.2c.5 0 .9-.4.9-.9v-10.2c0-.5-.4-.9-.9-.9H16a.5.5 0 0 1 0-1" clip-rule="evenodd"></path></svg></button></div></div></div></div></div></div></div></div></div></footer><div class="rk rl rm rn ro l"><div class="ab cb"><div class="ci bh fz ga gb gc"><div class="rp bh r rq"></div><div class="rr l"><div class="ab rs rt ru iz iy"><div class="rv rw rx ry rz sa sb sc sd se ab cp"><div class="h k"><a href="https://medium.com/outfit7?source=post_page---post_publication_info--c5fa1610d786--------------------------------" rel="noopener follow"><div class="fj ab"><img alt="Outfit7" class="sf ib ic cx" src="https://miro.medium.com/v2/resize:fill:96:96/1*dCIQWfMya9g57UUL2ZzrJQ.png" width="48" height="48" loading="lazy"/><div class="sf l ic ib fs n fr sg"></div></div></a></div><div class="j i d"><a href="https://medium.com/outfit7?source=post_page---post_publication_info--c5fa1610d786--------------------------------" rel="noopener follow"><div class="fj ab"><img alt="Outfit7" class="sf si sh cx" src="https://miro.medium.com/v2/resize:fill:128:128/1*dCIQWfMya9g57UUL2ZzrJQ.png" width="64" height="64" loading="lazy"/><div class="sf l sh si fs n fr sg"></div></div></a></div><div class="j i d sj ix"><div class="ab"></div></div></div><div class="ab co sk"><div class="sl sm sn so sp l"><a class="af ag ah aj ak al am an ao ap aq ar as at ab q" href="https://medium.com/outfit7?source=post_page---post_publication_info--c5fa1610d786--------------------------------" rel="noopener follow"><h2 class="pw-author-name bf sr ss st su sv sw sx nh qb qc nl qe qf np qh qi bk"><span class="gn sq">Published in Outfit7</span></h2></a><div class="qp ab ia"><div class="l ix"><span class="pw-follower-count bf b bg z du"><a class="af ag ah ai aj ak al am an ao ap aq ar iq" rel="noopener follow" href="/outfit7/followers?source=post_page---post_publication_info--c5fa1610d786--------------------------------">53 Followers</a></span></div><div class="bf b bg z du ab jd"><span class="ir l" aria-hidden="true"><span class="bf b bg z du">·</span></span><a class="af ag ah ai aj ak al am an ao ap aq ar iq" rel="noopener follow" href="/outfit7/new-home-for-outfit7-tech-articles-b317a75e1447?source=post_page---post_publication_info--c5fa1610d786--------------------------------">Last published <span>Sep 26, 2024</span></a></div></div><div class="sy l"><p class="bf b bg z bk"><span class="gn">The inside scoop on development, engineering & tech innovation @ Outfit7.</span></p></div></div></div><div class="h k"><div class="ab"></div></div></div></div><div class="ab rs rt ru iz iy"><div class="rv rw rx ry rz sa sb sc sd se ab cp"><div class="h k"><a tabindex="0" rel="noopener follow" href="/@tilen.kavcic_63182?source=post_page---post_author_info--c5fa1610d786--------------------------------"><div class="l fj"><img alt="Tilen Kavcic" class="l fd by ic ib cx" src="https://miro.medium.com/v2/resize:fill:96:96/0*huBuE1NwJmzOfK05" width="48" height="48" loading="lazy"/><div class="fr by l ic ib fs n ay sg"></div></div></a></div><div class="j i d"><a tabindex="0" rel="noopener follow" href="/@tilen.kavcic_63182?source=post_page---post_author_info--c5fa1610d786--------------------------------"><div class="l fj"><img alt="Tilen Kavcic" class="l fd by sh si cx" src="https://miro.medium.com/v2/resize:fill:128:128/0*huBuE1NwJmzOfK05" width="64" height="64" loading="lazy"/><div class="fr by l sh si fs n ay sg"></div></div></a></div><div class="j i d sj ix"><div class="ab"><span><button class="bf b bg z sz qr ta tb tc td te ev ew tf tg th fa fb fc fd bm fe ff">Follow</button></span></div></div></div><div class="ab co sk"><div class="sl sm sn so sp l"><a class="af ag ah aj ak al am an ao ap aq ar as at ab q" rel="noopener follow" href="/@tilen.kavcic_63182?source=post_page---post_author_info--c5fa1610d786--------------------------------"><h2 class="pw-author-name bf sr ss st su sv sw sx nh qb qc nl qe qf np qh qi bk"><span class="gn sq">Written by Tilen Kavcic</span></h2></a><div class="qp ab ia"><div class="l ix"><span class="pw-follower-count bf b bg z du"><a class="af ag ah ai aj ak al am an ao ap aq ar iq" rel="noopener follow" href="/@tilen.kavcic_63182/followers?source=post_page---post_author_info--c5fa1610d786--------------------------------">3 Followers</a></span></div><div class="bf b bg z du ab jd"><span class="ir l" aria-hidden="true"><span class="bf b bg z du">·</span></span><a class="af ag ah ai aj ak al am an ao ap aq ar iq" rel="noopener follow" href="/@tilen.kavcic_63182/following?source=post_page---post_author_info--c5fa1610d786--------------------------------">1 Following</a></div></div><div class="sy l"></div></div></div><div class="h k"><div class="ab"><span><button class="bf b bg z sz qr ta tb tc td te ev ew tf tg th fa fb fc fd bm fe ff">Follow</button></span></div></div></div></div></div></div><div class="ti l"><div class="rp bh r tj tk tl tm tn"></div><div class="ab cb"><div class="ci bh fz ga gb gc"><div class="ab q cp"><h2 class="bf sr om oo op oq os ot ou ow ox oy pa pb pc pe pf bk">No responses yet</h2><div class="ab to"><div><div class="bm" aria-hidden="false"><a class="tp tq" href="https://policy.medium.com/medium-rules-30e5502c4eb4?source=post_page---post_responses--c5fa1610d786--------------------------------" rel="noopener follow" target="_blank"><svg xmlns="http://www.w3.org/2000/svg" width="25" height="25" viewBox="0 0 25 25"><path fill-rule="evenodd" d="M11.987 5.036a.754.754 0 0 1 .914-.01c.972.721 1.767 1.218 2.6 1.543.828.322 1.719.485 2.887.505a.755.755 0 0 1 .741.757c-.018 3.623-.43 6.256-1.449 8.21-1.034 1.984-2.662 3.209-4.966 4.083a.75.75 0 0 1-.537-.003c-2.243-.874-3.858-2.095-4.897-4.074-1.024-1.951-1.457-4.583-1.476-8.216a.755.755 0 0 1 .741-.757c1.195-.02 2.1-.182 2.923-.503.827-.322 1.6-.815 2.519-1.535m.468.903c-.897.69-1.717 1.21-2.623 1.564-.898.35-1.856.527-3.026.565.037 3.45.469 5.817 1.36 7.515.884 1.684 2.25 2.762 4.284 3.571 2.092-.81 3.465-1.89 4.344-3.575.886-1.698 1.299-4.065 1.334-7.512-1.149-.039-2.091-.217-2.99-.567-.906-.353-1.745-.873-2.683-1.561m-.009 9.155a2.672 2.672 0 1 0 0-5.344 2.672 2.672 0 0 0 0 5.344m0 1a3.672 3.672 0 1 0 0-7.344 3.672 3.672 0 0 0 0 7.344m-1.813-3.777.525-.526.916.917 1.623-1.625.526.526-2.149 2.152z" clip-rule="evenodd"></path></svg></a></div></div></div></div></div></div></div><div class="tr ts tt tu tv l bx"><div class="h k j"><div class="rp bh tw tx"></div><div class="ab cb"><div class="ci bh fz ga gb gc"><div class="ty ab kv ja"><div class="tz ua l"><a class="af ag ah ai aj ak al am an ao ap aq ar as at" href="https://help.medium.com/hc/en-us?source=post_page-----c5fa1610d786--------------------------------" rel="noopener follow"><p class="bf b dv z du">Help</p></a></div><div class="tz ua l"><a class="af ag ah ai aj ak al am an ao ap aq ar as at" href="https://medium.statuspage.io/?source=post_page-----c5fa1610d786--------------------------------" rel="noopener follow"><p class="bf b dv z du">Status</p></a></div><div class="tz ua l"><a class="af ag ah ai aj ak al am an ao ap aq ar as at" rel="noopener follow" href="/about?autoplay=1&source=post_page-----c5fa1610d786--------------------------------"><p class="bf b dv z du">About</p></a></div><div class="tz ua l"><a class="af ag ah ai aj ak al am an ao ap aq ar as at" rel="noopener follow" href="/jobs-at-medium/work-at-medium-959d1a85284e?source=post_page-----c5fa1610d786--------------------------------"><p class="bf b dv z du">Careers</p></a></div><div class="tz ua l"><a class="af ag ah ai aj ak al am an ao ap aq ar as at" href="pressinquiries@medium.com?source=post_page-----c5fa1610d786--------------------------------" rel="noopener follow"><p class="bf b dv z du">Press</p></a></div><div class="tz ua l"><a class="af ag ah ai aj ak al am an ao ap aq ar as at" href="https://blog.medium.com/?source=post_page-----c5fa1610d786--------------------------------" rel="noopener follow"><p class="bf b dv z du">Blog</p></a></div><div class="tz ua l"><a class="af ag ah ai aj ak al am an ao ap aq ar as at" href="https://policy.medium.com/medium-privacy-policy-f03bf92035c9?source=post_page-----c5fa1610d786--------------------------------" rel="noopener follow"><p class="bf b dv z du">Privacy</p></a></div><div class="tz ua l"><a class="af ag ah ai aj ak al am an ao ap aq ar as at" href="https://policy.medium.com/medium-terms-of-service-9db0094a1e0f?source=post_page-----c5fa1610d786--------------------------------" rel="noopener follow"><p class="bf b dv z du">Terms</p></a></div><div class="tz ua l"><a class="af ag ah ai aj ak al am an ao ap aq ar as at" href="https://speechify.com/medium?source=post_page-----c5fa1610d786--------------------------------" rel="noopener follow"><p class="bf b dv z du">Text to speech</p></a></div><div class="tz l"><a class="af ag ah ai aj ak al am an ao ap aq ar as at" rel="noopener follow" href="/business?source=post_page-----c5fa1610d786--------------------------------"><p class="bf b dv z du">Teams</p></a></div></div></div></div></div></div></div></div></div></div><script>window.__BUILD_ID__="main-20241122-185319-7bcdc08639"</script><script>window.__GRAPHQL_URI__ = "https://medium.com/_/graphql"</script><script>window.__PRELOADED_STATE__ = {"algolia":{"queries":{}},"cache":{"experimentGroupSet":true,"reason":"","group":"enabled","tags":["group-edgeCachePosts","post-c5fa1610d786","user-c7d7326118b3","collection-6ec626d5ad62"],"serverVariantState":"44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a","middlewareEnabled":true,"cacheStatus":"DYNAMIC","shouldUseCache":true,"vary":[],"lohpSummerUpsellEnabled":false,"publicationHierarchyEnabledWeb":false,"postBottomResponsesEnabled":false},"client":{"hydrated":false,"isUs":false,"isNativeMedium":false,"isSafariMobile":false,"isSafari":false,"isFirefox":false,"routingEntity":{"type":"DEFAULT","explicit":false},"viewerIsBot":false},"debug":{"requestId":"6b9a144e-4db2-48a9-b7ad-3ac1e07cd54c","hybridDevServices":[],"originalSpanCarrier":{"traceparent":"00-e9f179b2a405baa7761517f2793bfea6-6be972d991dec8e0-01"}},"multiVote":{"clapsPerPost":{}},"navigation":{"branch":{"show":null,"hasRendered":null,"blockedByCTA":false},"hideGoogleOneTap":false,"hasRenderedAlternateUserBanner":null,"currentLocation":"https:\u002F\u002Fmedium.com\u002Foutfit7\u002Fthe-billion-events-infrastructure-c5fa1610d786","host":"medium.com","hostname":"medium.com","referrer":"","hasSetReferrer":false,"susiModal":{"step":null,"operation":"register"},"postRead":false,"partnerProgram":{"selectedCountryCode":null},"queryString":"","currentHash":""},"config":{"nodeEnv":"production","version":"main-20241122-185319-7bcdc08639","target":"production","productName":"Medium","publicUrl":"https:\u002F\u002Fcdn-client.medium.com\u002Flite","authDomain":"medium.com","authGoogleClientId":"216296035834-k1k6qe060s2tp2a2jam4ljdcms00sttg.apps.googleusercontent.com","favicon":"production","glyphUrl":"https:\u002F\u002Fglyph.medium.com","branchKey":"key_live_ofxXr2qTrrU9NqURK8ZwEhknBxiI6KBm","algolia":{"appId":"MQ57UUUQZ2","apiKeySearch":"394474ced050e3911ae2249ecc774921","indexPrefix":"medium_","host":"-dsn.algolia.net"},"recaptchaKey":"6Lfc37IUAAAAAKGGtC6rLS13R1Hrw_BqADfS1LRk","recaptcha3Key":"6Lf8R9wUAAAAABMI_85Wb8melS7Zj6ziuf99Yot5","recaptchaEnterpriseKeyId":"6Le-uGgpAAAAAPprRaokM8AKthQ9KNGdoxaGUvVp","datadog":{"applicationId":"6702d87d-a7e0-42fe-bbcb-95b469547ea0","clientToken":"pub853ea8d17ad6821d9f8f11861d23dfed","rumToken":"pubf9cc52896502b9413b68ba36fc0c7162","context":{"deployment":{"target":"production","tag":"main-20241122-185319-7bcdc08639","commit":"7bcdc08639c179dc5172558201a3fd3abc1b5db6"}},"datacenter":"us"},"googleAnalyticsCode":"G-7JY7T788PK","googlePay":{"apiVersion":"2","apiVersionMinor":"0","merchantId":"BCR2DN6TV7EMTGBM","merchantName":"Medium","instanceMerchantId":"13685562959212738550"},"applePay":{"version":3},"signInWallCustomDomainCollectionIds":["3a8144eabfe3","336d898217ee","61061eb0c96b","138adf9c44c","819cc2aaeee0"],"mediumMastodonDomainName":"me.dm","mediumOwnedAndOperatedCollectionIds":["8a9336e5bb4","b7e45b22fec3","193b68bd4fba","8d6b8a439e32","54c98c43354d","3f6ecf56618","d944778ce714","92d2092dc598","ae2a65f35510","1285ba81cada","544c7006046e","fc8964313712","40187e704f1c","88d9857e584e","7b6769f2748b","bcc38c8f6edf","cef6983b292","cb8577c9149e","444d13b52878","713d7dbc99b0","ef8e90590e66","191186aaafa0","55760f21cdc5","9dc80918cc93","bdc4052bbdba","8ccfed20cbb2"],"tierOneDomains":["medium.com","thebolditalic.com","arcdigital.media","towardsdatascience.com","uxdesign.cc","codeburst.io","psiloveyou.xyz","writingcooperative.com","entrepreneurshandbook.co","prototypr.io","betterhumans.coach.me","theascent.pub"],"topicsToFollow":["d61cf867d93f","8a146bc21b28","1eca0103fff3","4d562ee63426","aef1078a3ef5","e15e46793f8d","6158eb913466","55f1c20aba7a","3d18b94f6858","4861fee224fd","63c6f1f93ee","1d98b3a9a871","decb52b64abf","ae5d4995e225","830cded25262"],"topicToTagMappings":{"accessibility":"accessibility","addiction":"addiction","android-development":"android-development","art":"art","artificial-intelligence":"artificial-intelligence","astrology":"astrology","basic-income":"basic-income","beauty":"beauty","biotech":"biotech","blockchain":"blockchain","books":"books","business":"business","cannabis":"cannabis","cities":"cities","climate-change":"climate-change","comics":"comics","coronavirus":"coronavirus","creativity":"creativity","cryptocurrency":"cryptocurrency","culture":"culture","cybersecurity":"cybersecurity","data-science":"data-science","design":"design","digital-life":"digital-life","disability":"disability","economy":"economy","education":"education","equality":"equality","family":"family","feminism":"feminism","fiction":"fiction","film":"film","fitness":"fitness","food":"food","freelancing":"freelancing","future":"future","gadgets":"gadgets","gaming":"gaming","gun-control":"gun-control","health":"health","history":"history","humor":"humor","immigration":"immigration","ios-development":"ios-development","javascript":"javascript","justice":"justice","language":"language","leadership":"leadership","lgbtqia":"lgbtqia","lifestyle":"lifestyle","machine-learning":"machine-learning","makers":"makers","marketing":"marketing","math":"math","media":"media","mental-health":"mental-health","mindfulness":"mindfulness","money":"money","music":"music","neuroscience":"neuroscience","nonfiction":"nonfiction","outdoors":"outdoors","parenting":"parenting","pets":"pets","philosophy":"philosophy","photography":"photography","podcasts":"podcast","poetry":"poetry","politics":"politics","privacy":"privacy","product-management":"product-management","productivity":"productivity","programming":"programming","psychedelics":"psychedelics","psychology":"psychology","race":"race","relationships":"relationships","religion":"religion","remote-work":"remote-work","san-francisco":"san-francisco","science":"science","self":"self","self-driving-cars":"self-driving-cars","sexuality":"sexuality","social-media":"social-media","society":"society","software-engineering":"software-engineering","space":"space","spirituality":"spirituality","sports":"sports","startups":"startup","style":"style","technology":"technology","transportation":"transportation","travel":"travel","true-crime":"true-crime","tv":"tv","ux":"ux","venture-capital":"venture-capital","visual-design":"visual-design","work":"work","world":"world","writing":"writing"},"defaultImages":{"avatar":{"imageId":"1*dmbNkD5D-u45r44go_cf0g.png","height":150,"width":150},"orgLogo":{"imageId":"7*V1_7XP4snlmqrc_0Njontw.png","height":110,"width":500},"postLogo":{"imageId":"bd978bb536350a710e8efb012513429cabdc4c28700604261aeda246d0f980b7","height":810,"width":1440},"postPreviewImage":{"imageId":"1*hn4v1tCaJy7cWMyb0bpNpQ.png","height":386,"width":579}},"collectionStructuredData":{"8d6b8a439e32":{"name":"Elemental","data":{"@type":"NewsMediaOrganization","ethicsPolicy":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Farticles\u002F360043290473","logo":{"@type":"ImageObject","url":"https:\u002F\u002Fcdn-images-1.medium.com\u002Fmax\u002F980\u002F1*9ygdqoKprhwuTVKUM0DLPA@2x.png","width":980,"height":159}}},"3f6ecf56618":{"name":"Forge","data":{"@type":"NewsMediaOrganization","ethicsPolicy":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Farticles\u002F360043290473","logo":{"@type":"ImageObject","url":"https:\u002F\u002Fcdn-images-1.medium.com\u002Fmax\u002F596\u002F1*uULpIlImcO5TDuBZ6lm7Lg@2x.png","width":596,"height":183}}},"ae2a65f35510":{"name":"GEN","data":{"@type":"NewsMediaOrganization","ethicsPolicy":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Farticles\u002F360043290473","logo":{"@type":"ImageObject","url":"https:\u002F\u002Fmiro.medium.com\u002Fmax\u002F264\u002F1*RdVZMdvfV3YiZTw6mX7yWA.png","width":264,"height":140}}},"88d9857e584e":{"name":"LEVEL","data":{"@type":"NewsMediaOrganization","ethicsPolicy":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Farticles\u002F360043290473","logo":{"@type":"ImageObject","url":"https:\u002F\u002Fmiro.medium.com\u002Fmax\u002F540\u002F1*JqYMhNX6KNNb2UlqGqO2WQ.png","width":540,"height":108}}},"7b6769f2748b":{"name":"Marker","data":{"@type":"NewsMediaOrganization","ethicsPolicy":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Farticles\u002F360043290473","logo":{"@type":"ImageObject","url":"https:\u002F\u002Fcdn-images-1.medium.com\u002Fmax\u002F383\u002F1*haCUs0wF6TgOOvfoY-jEoQ@2x.png","width":383,"height":92}}},"444d13b52878":{"name":"OneZero","data":{"@type":"NewsMediaOrganization","ethicsPolicy":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Farticles\u002F360043290473","logo":{"@type":"ImageObject","url":"https:\u002F\u002Fmiro.medium.com\u002Fmax\u002F540\u002F1*cw32fIqCbRWzwJaoQw6BUg.png","width":540,"height":123}}},"8ccfed20cbb2":{"name":"Zora","data":{"@type":"NewsMediaOrganization","ethicsPolicy":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Farticles\u002F360043290473","logo":{"@type":"ImageObject","url":"https:\u002F\u002Fmiro.medium.com\u002Fmax\u002F540\u002F1*tZUQqRcCCZDXjjiZ4bDvgQ.png","width":540,"height":106}}}},"embeddedPostIds":{"coronavirus":"cd3010f9d81f"},"sharedCdcMessaging":{"COVID_APPLICABLE_TAG_SLUGS":[],"COVID_APPLICABLE_TOPIC_NAMES":[],"COVID_APPLICABLE_TOPIC_NAMES_FOR_TOPIC_PAGE":[],"COVID_MESSAGES":{"tierA":{"text":"For more information on the novel coronavirus and Covid-19, visit cdc.gov.","markups":[{"start":66,"end":73,"href":"https:\u002F\u002Fwww.cdc.gov\u002Fcoronavirus\u002F2019-nCoV"}]},"tierB":{"text":"Anyone can publish on Medium per our Policies, but we don’t fact-check every story. For more info about the coronavirus, see cdc.gov.","markups":[{"start":37,"end":45,"href":"https:\u002F\u002Fhelp.medium.com\u002Fhc\u002Fen-us\u002Fcategories\u002F201931128-Policies-Safety"},{"start":125,"end":132,"href":"https:\u002F\u002Fwww.cdc.gov\u002Fcoronavirus\u002F2019-nCoV"}]},"paywall":{"text":"This article has been made free for everyone, thanks to Medium Members. For more information on the novel coronavirus and Covid-19, visit cdc.gov.","markups":[{"start":56,"end":70,"href":"https:\u002F\u002Fmedium.com\u002Fmembership"},{"start":138,"end":145,"href":"https:\u002F\u002Fwww.cdc.gov\u002Fcoronavirus\u002F2019-nCoV"}]},"unbound":{"text":"This article is free for everyone, thanks to Medium Members. For more information on the novel coronavirus and Covid-19, visit cdc.gov.","markups":[{"start":45,"end":59,"href":"https:\u002F\u002Fmedium.com\u002Fmembership"},{"start":127,"end":134,"href":"https:\u002F\u002Fwww.cdc.gov\u002Fcoronavirus\u002F2019-nCoV"}]}},"COVID_BANNER_POST_ID_OVERRIDE_WHITELIST":["3b31a67bff4a"]},"sharedVoteMessaging":{"TAGS":["politics","election-2020","government","us-politics","election","2020-presidential-race","trump","donald-trump","democrats","republicans","congress","republican-party","democratic-party","biden","joe-biden","maga"],"TOPICS":["politics","election"],"MESSAGE":{"text":"Find out more about the U.S. election results here.","markups":[{"start":46,"end":50,"href":"https:\u002F\u002Fcookpolitical.com\u002F2020-national-popular-vote-tracker"}]},"EXCLUDE_POSTS":["397ef29e3ca5"]},"embedPostRules":[],"recircOptions":{"v1":{"limit":3},"v2":{"limit":8}},"braintreeClientKey":"production_zjkj96jm_m56f8fqpf7ngnrd4","braintree":{"enabled":true,"merchantId":"m56f8fqpf7ngnrd4","merchantAccountId":{"usd":"AMediumCorporation_instant","eur":"amediumcorporation_EUR","cad":"amediumcorporation_CAD"},"publicKey":"ds2nn34bg2z7j5gd","braintreeEnvironment":"production","dashboardUrl":"https:\u002F\u002Fwww.braintreegateway.com\u002Fmerchants","gracePeriodDurationInDays":14,"mediumMembershipPlanId":{"monthly":"ce105f8c57a3","monthlyV2":"e8a5e126-792b-4ee6-8fba-d574c1b02fc5","monthlyWithTrial":"d5ee3dbe3db8","monthlyPremium":"fa741a9b47a2","yearly":"a40ad4a43185","yearlyV2":"3815d7d6-b8ca-4224-9b8c-182f9047866e","yearlyStaff":"d74fb811198a","yearlyWithTrial":"b3bc7350e5c7","yearlyPremium":"e21bd2c12166","monthlyOneYearFree":"e6c0637a-2bad-4171-ab4f-3c268633d83c","monthly25PercentOffFirstYear":"235ecc62-0cdb-49ae-9378-726cd21c504b","monthly20PercentOffFirstYear":"ba518864-9c13-4a99-91ca-411bf0cac756","monthly15PercentOffFirstYear":"594c029b-9f89-43d5-88f8-8173af4e070e","monthly10PercentOffFirstYear":"c6c7bc9a-40f2-4b51-8126-e28511d5bdb0","monthlyForStudents":"629ebe51-da7d-41fd-8293-34cd2f2030a8","yearlyOneYearFree":"78ba7be9-0d9f-4ece-aa3e-b54b826f2bf1","yearly25PercentOffFirstYear":"2dbb010d-bb8f-4eeb-ad5c-a08509f42d34","yearly20PercentOffFirstYear":"47565488-435b-47f8-bf93-40d5fbe0ebc8","yearly15PercentOffFirstYear":"8259809b-0881-47d9-acf7-6c001c7f720f","yearly10PercentOffFirstYear":"9dd694fb-96e1-472c-8d9e-3c868d5c1506","yearlyForStudents":"e29345ef-ab1c-4234-95c5-70e50fe6bc23","monthlyCad":"p52orjkaceei","yearlyCad":"h4q9g2up9ktt"},"braintreeDiscountId":{"oneMonthFree":"MONTHS_FREE_01","threeMonthsFree":"MONTHS_FREE_03","sixMonthsFree":"MONTHS_FREE_06","fiftyPercentOffOneYear":"FIFTY_PERCENT_OFF_ONE_YEAR"},"3DSecureVersion":"2","defaultCurrency":"usd","providerPlanIdCurrency":{"4ycw":"usd","rz3b":"usd","3kqm":"usd","jzw6":"usd","c2q2":"usd","nnsw":"usd","q8qw":"usd","d9y6":"usd","fx7w":"cad","nwf2":"cad"}},"paypalClientId":"AXj1G4fotC2GE8KzWX9mSxCH1wmPE3nJglf4Z2ig_amnhvlMVX87otaq58niAg9iuLktVNF_1WCMnN7v","paypal":{"host":"https:\u002F\u002Fapi.paypal.com:443","clientMode":"production","serverMode":"live","webhookId":"4G466076A0294510S","monthlyPlan":{"planId":"P-9WR0658853113943TMU5FDQA","name":"Medium Membership (Monthly) with setup fee","description":"Unlimited access to the best and brightest stories on Medium. Membership billed monthly."},"yearlyPlan":{"planId":"P-7N8963881P8875835MU5JOPQ","name":"Medium Membership (Annual) with setup fee","description":"Unlimited access to the best and brightest stories on Medium. Membership billed annually."},"oneYearGift":{"name":"Medium Membership (1 Year, Digital Gift Code)","description":"Unlimited access to the best and brightest stories on Medium. Gift codes can be redeemed at medium.com\u002Fredeem.","price":"50.00","currency":"USD","sku":"membership-gift-1-yr"},"oldMonthlyPlan":{"planId":"P-96U02458LM656772MJZUVH2Y","name":"Medium Membership (Monthly)","description":"Unlimited access to the best and brightest stories on Medium. Membership billed monthly."},"oldYearlyPlan":{"planId":"P-59P80963JF186412JJZU3SMI","name":"Medium Membership (Annual)","description":"Unlimited access to the best and brightest stories on Medium. Membership billed annually."},"monthlyPlanWithTrial":{"planId":"P-66C21969LR178604GJPVKUKY","name":"Medium Membership (Monthly) with setup fee","description":"Unlimited access to the best and brightest stories on Medium. Membership billed monthly."},"yearlyPlanWithTrial":{"planId":"P-6XW32684EX226940VKCT2MFA","name":"Medium Membership (Annual) with setup fee","description":"Unlimited access to the best and brightest stories on Medium. Membership billed annually."},"oldMonthlyPlanNoSetupFee":{"planId":"P-4N046520HR188054PCJC7LJI","name":"Medium Membership (Monthly)","description":"Unlimited access to the best and brightest stories on Medium. Membership billed monthly."},"oldYearlyPlanNoSetupFee":{"planId":"P-7A4913502Y5181304CJEJMXQ","name":"Medium Membership (Annual)","description":"Unlimited access to the best and brightest stories on Medium. Membership billed annually."},"sdkUrl":"https:\u002F\u002Fwww.paypal.com\u002Fsdk\u002Fjs"},"stripePublishableKey":"pk_live_7FReX44VnNIInZwrIIx6ghjl","log":{"json":true,"level":"info"},"imageUploadMaxSizeMb":25,"staffPicks":{"title":"Staff Picks","catalogId":"c7bc6e1ee00f"}},"session":{"xsrf":""}}</script><script>window.__APOLLO_STATE__ = {"ImageMetadata:":{"__typename":"ImageMetadata","id":""},"Collection:6ec626d5ad62":{"__typename":"Collection","id":"6ec626d5ad62","favicon":{"__ref":"ImageMetadata:"},"customStyleSheet":null,"colorPalette":{"__typename":"ColorPalette","highlightSpectrum":{"__typename":"ColorSpectrum","backgroundColor":"#FFFFFFFF","colorPoints":[{"__typename":"ColorPoint","color":"#FFF5F2F1","point":0},{"__typename":"ColorPoint","color":"#FFF3F0EF","point":0.1},{"__typename":"ColorPoint","color":"#FFF1EEEE","point":0.2},{"__typename":"ColorPoint","color":"#FFEFECEC","point":0.3},{"__typename":"ColorPoint","color":"#FFEDEAEA","point":0.4},{"__typename":"ColorPoint","color":"#FFEBE8E8","point":0.5},{"__typename":"ColorPoint","color":"#FFE9E7E6","point":0.6},{"__typename":"ColorPoint","color":"#FFE7E5E4","point":0.7},{"__typename":"ColorPoint","color":"#FFE5E3E2","point":0.8},{"__typename":"ColorPoint","color":"#FFE3E1E0","point":0.9},{"__typename":"ColorPoint","color":"#FFE1DFDE","point":1}]},"defaultBackgroundSpectrum":{"__typename":"ColorSpectrum","backgroundColor":"#FFFFFFFF","colorPoints":[{"__typename":"ColorPoint","color":"#FF858484","point":0},{"__typename":"ColorPoint","color":"#FF7C7B7B","point":0.1},{"__typename":"ColorPoint","color":"#FF737171","point":0.2},{"__typename":"ColorPoint","color":"#FF696868","point":0.3},{"__typename":"ColorPoint","color":"#FF5F5E5E","point":0.4},{"__typename":"ColorPoint","color":"#FF555454","point":0.5},{"__typename":"ColorPoint","color":"#FF4A4949","point":0.6},{"__typename":"ColorPoint","color":"#FF3F3E3E","point":0.7},{"__typename":"ColorPoint","color":"#FF343333","point":0.8},{"__typename":"ColorPoint","color":"#FF272727","point":0.9},{"__typename":"ColorPoint","color":"#FF1A1A1A","point":1}]},"tintBackgroundSpectrum":{"__typename":"ColorSpectrum","backgroundColor":"#FF111111","colorPoints":[{"__typename":"ColorPoint","color":"#FF111111","point":0},{"__typename":"ColorPoint","color":"#FF313131","point":0.1},{"__typename":"ColorPoint","color":"#FF4D4C4C","point":0.2},{"__typename":"ColorPoint","color":"#FF666565","point":0.3},{"__typename":"ColorPoint","color":"#FF7D7C7C","point":0.4},{"__typename":"ColorPoint","color":"#FF939292","point":0.5},{"__typename":"ColorPoint","color":"#FFA9A7A7","point":0.6},{"__typename":"ColorPoint","color":"#FFBDBBBB","point":0.7},{"__typename":"ColorPoint","color":"#FFD2CFCF","point":0.8},{"__typename":"ColorPoint","color":"#FFE5E2E2","point":0.9},{"__typename":"ColorPoint","color":"#FFF8F5F5","point":1}]}},"domain":null,"slug":"outfit7","googleAnalyticsId":null,"editors":[{"__typename":"CollectionMastheadUserItem","user":{"__ref":"User:ebaddf6e1f2a"}}],"name":"Outfit7","avatar":{"__ref":"ImageMetadata:1*dCIQWfMya9g57UUL2ZzrJQ.png"},"description":"The inside scoop on development, engineering & tech innovation @ Outfit7.","subscriberCount":53,"latestPostsConnection({\"paging\":{\"limit\":1}})":{"__typename":"PostConnection","posts":[{"__ref":"Post:b317a75e1447"}]},"viewerEdge":{"__ref":"CollectionViewerEdge:collectionId:6ec626d5ad62-viewerId:lo_08ce0b10983d"},"twitterUsername":null,"facebookPageId":null,"logo":{"__ref":"ImageMetadata:1*nTxl_7KgWQYo8PfVROUK_A.png"}},"ROOT_QUERY":{"__typename":"Query","collectionByDomainOrSlug({\"domainOrSlug\":\"outfit7\"})":{"__ref":"Collection:6ec626d5ad62"},"viewer":null,"postResult({\"id\":\"c5fa1610d786\"})":{"__ref":"Post:c5fa1610d786"}},"User:ebaddf6e1f2a":{"__typename":"User","id":"ebaddf6e1f2a"},"ImageMetadata:1*dCIQWfMya9g57UUL2ZzrJQ.png":{"__typename":"ImageMetadata","id":"1*dCIQWfMya9g57UUL2ZzrJQ.png"},"User:3215f0e451af":{"__typename":"User","id":"3215f0e451af","customDomainState":null,"hasSubdomain":false,"username":"jure.gorinsek_80693"},"Post:b317a75e1447":{"__typename":"Post","id":"b317a75e1447","firstPublishedAt":1727352997939,"creator":{"__ref":"User:3215f0e451af"},"collection":{"__ref":"Collection:6ec626d5ad62"},"isSeries":false,"mediumUrl":"https:\u002F\u002Fmedium.com\u002Foutfit7\u002Fnew-home-for-outfit7-tech-articles-b317a75e1447","sequence":null,"uniqueSlug":"new-home-for-outfit7-tech-articles-b317a75e1447"},"LinkedAccounts:c7d7326118b3":{"__typename":"LinkedAccounts","mastodon":null,"id":"c7d7326118b3"},"UserViewerEdge:userId:c7d7326118b3-viewerId:lo_08ce0b10983d":{"__typename":"UserViewerEdge","id":"userId:c7d7326118b3-viewerId:lo_08ce0b10983d","isFollowing":false,"isUser":false,"isMuting":false},"User:c7d7326118b3":{"__typename":"User","id":"c7d7326118b3","linkedAccounts":{"__ref":"LinkedAccounts:c7d7326118b3"},"isSuspended":false,"imageId":"0*huBuE1NwJmzOfK05","mediumMemberAt":0,"verifications":{"__typename":"VerifiedInfo","isBookAuthor":false},"name":"Tilen Kavcic","socialStats":{"__typename":"SocialStats","followerCount":3,"followingCount":1,"collectionFollowingCount":0},"username":"tilen.kavcic_63182","customDomainState":null,"hasSubdomain":false,"bio":"","isPartnerProgramEnrolled":false,"viewerEdge":{"__ref":"UserViewerEdge:userId:c7d7326118b3-viewerId:lo_08ce0b10983d"},"viewerIsUser":false,"newsletterV3":null,"postSubscribeMembershipUpsellShownAt":0,"membership":null,"allowNotes":true,"twitterScreenName":""},"Topic:decb52b64abf":{"__typename":"Topic","slug":"programming","id":"decb52b64abf","name":"Programming"},"Paragraph:e585aa5e6f5b_0":{"__typename":"Paragraph","id":"e585aa5e6f5b_0","name":"02c9","type":"H3","href":null,"layout":null,"metadata":null,"text":"The Billion Events Infrastructure","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_1":{"__typename":"Paragraph","id":"e585aa5e6f5b_1","name":"858a","type":"P","href":null,"layout":null,"metadata":null,"text":"TL;DR: Outfit7 is, at its core, a data-driven company. Collecting and analyzing performance metrics from our games holds a lot of interesting challenges. In this article, I’ll explore our infrastructure design decisions and what we have learned along the way.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"STRONG","start":0,"end":7,"href":null,"anchorType":null,"userId":null,"linkMetadata":null},{"__typename":"Markup","type":"EM","start":0,"end":259,"href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"ImageMetadata:0*YpS6May9aFFtbR0D":{"__typename":"ImageMetadata","id":"0*YpS6May9aFFtbR0D","originalHeight":838,"originalWidth":1600,"focusPercentX":null,"focusPercentY":null,"alt":null},"Paragraph:e585aa5e6f5b_2":{"__typename":"Paragraph","id":"e585aa5e6f5b_2","name":"a879","type":"IMG","href":null,"layout":"INSET_CENTER","metadata":{"__ref":"ImageMetadata:0*YpS6May9aFFtbR0D"},"text":"","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_3":{"__typename":"Paragraph","id":"e585aa5e6f5b_3","name":"3ee6","type":"P","href":null,"layout":null,"metadata":null,"text":"Hi, my name is Tilen Kavčič. I’m one of the backend core members at Outfit7, specializing in cloud infrastructure and R&D. Our team’s mission is to create high performance, scalable solutions that support the growth of our company. I’m part software engineer, part data engineer, with an interest in DevOps. The best part of my job is getting the chance to create solutions that simplify development of new software. My current mission is to modernize our data stack, bringing software engineering best practices to our data teams.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_4":{"__typename":"Paragraph","id":"e585aa5e6f5b_4","name":"d5e6","type":"H3","href":null,"layout":null,"metadata":null,"text":"Background","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_5":{"__typename":"Paragraph","id":"e585aa5e6f5b_5","name":"1dbe","type":"P","href":null,"layout":null,"metadata":null,"text":"At Outfit7, we believe that all our games should look, play and feel good. With 430 million monthly active users, it can be a challenge to get a clear picture of how our games and company are performing. The data we gather helps us understand how we perform in the fast-paced mobile gaming market. Getting fast feedback is key. Because of this, we need infrastructure that is scalable and reliable, and which, ideally, doesn’t hurt the company’s wallet too much. Over the past decade we’ve continuously improved our infrastructure to accommodate our growing user base. Today, we handle half a million row inserts per second — let me show you how. Welcome to the Billion Events Infrastructure.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_6":{"__typename":"Paragraph","id":"e585aa5e6f5b_6","name":"97b5","type":"H3","href":null,"layout":null,"metadata":null,"text":"Infrastructure","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"ImageMetadata:0*Zek8d75_0P4uhYkI":{"__typename":"ImageMetadata","id":"0*Zek8d75_0P4uhYkI","originalHeight":301,"originalWidth":861,"focusPercentX":null,"focusPercentY":null,"alt":null},"Paragraph:e585aa5e6f5b_7":{"__typename":"Paragraph","id":"e585aa5e6f5b_7","name":"a04e","type":"IMG","href":null,"layout":"INSET_CENTER","metadata":{"__ref":"ImageMetadata:0*Zek8d75_0P4uhYkI"},"text":"","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_8":{"__typename":"Paragraph","id":"e585aa5e6f5b_8","name":"2381","type":"P","href":null,"layout":null,"metadata":null,"text":"Let’s start with an overview of our infrastructure. We receive data from various data sources, like game usage data, third party services, etc., which we save in persistent storage. Here, we’ll focus only on data sent by our games, because this is what forms the backbone of our analyses. From data gathering endpoints to persistent storage, all is hosted on the Google Cloud Platform. Let’s start with the first layer, at our API endpoints:","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_9":{"__typename":"Paragraph","id":"e585aa5e6f5b_9","name":"7759","type":"OLI","href":null,"layout":null,"metadata":null,"text":"API endpoints\nWe run our data-gathering endpoints on different managed services (App Engine, Cloud Run and Kubernetes Engine). Our main endpoint, where we gather usage data, runs on Kubernetes Engine. We chose Kubernetes because it offers a nice balance between performance and cost. Because we have a global player base, we run three clusters, each in different regions (US, EU and Asia). Our main cluster resides in the US, where we get the most traffic, while the Asia cluster serves as the endpoint for our users in China. We also set up one cluster in Europe which is our backfall endpoint in the event of endpoint failures in US and Asia clusters. All data is validated and pushed to our data delivery layer.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"STRONG","start":0,"end":13,"href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_10":{"__typename":"Paragraph","id":"e585aa5e6f5b_10","name":"bf36","type":"OLI","href":null,"layout":null,"metadata":null,"text":"Data delivery\nThis second layer is a middleware between our API endpoints and data processing workloads. It provides asynchronous delivery of messages with the benefit of keeping our data safe until it’s written to persistent storage. We use Pub\u002FSub (similar to Apache Kafka) which provides a simple, reliable and scalable message delivery service.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"STRONG","start":0,"end":13,"href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_11":{"__typename":"Paragraph","id":"e585aa5e6f5b_11","name":"943f","type":"OLI","href":null,"layout":null,"metadata":null,"text":"Data processing\nIn this layer, we do some light data transformation (unbatching the client data) and streaming data into persistent storage. Our main framework to do this is Apache Beam which runs on managed cloud infrastructure called Dataflow. This is a tried and tested way to do ETL pipelines because it provides a reliable way of inserting data into persistent storage.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"STRONG","start":0,"end":16,"href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_12":{"__typename":"Paragraph","id":"e585aa5e6f5b_12","name":"dac0","type":"OLI","href":null,"layout":null,"metadata":null,"text":"Data storage\nAfter all this, the data is finally written into BigQuery. It’s our main data warehouse, which powers all of our analytics workload. It’s fast, reliable and just what we need to store and analyze our data.","hasDropCap":null,"dropCapImage":null,"markups":[{"__typename":"Markup","type":"STRONG","start":0,"end":13,"href":null,"anchorType":null,"userId":null,"linkMetadata":null}],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_13":{"__typename":"Paragraph","id":"e585aa5e6f5b_13","name":"8efe","type":"P","href":null,"layout":null,"metadata":null,"text":"All these layers combined create a scalable and reliable infrastructure that can receive, process and ingest more than half a million rows of data per second, or five terabytes of new data per day. Our data teams query 60 petabytes per year with zero downtime and zero maintenance from the core engineering team, just by using BigQuery. By using a simple approach and managed services a single person can maintain this infrastructure. This enables us to focus on developing new solutions that add value to our company.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_14":{"__typename":"Paragraph","id":"e585aa5e6f5b_14","name":"e88e","type":"H3","href":null,"layout":null,"metadata":null,"text":"What we learned","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_15":{"__typename":"Paragraph","id":"e585aa5e6f5b_15","name":"ef35","type":"H4","href":null,"layout":null,"metadata":null,"text":"Development environment","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_16":{"__typename":"Paragraph","id":"e585aa5e6f5b_16","name":"aa72","type":"P","href":null,"layout":null,"metadata":null,"text":"These environments are important because they create a safe space where engineers can create and test new features. Without them you are always one step away from breaking the production environment. Our team takes these environments seriously. At the very beginning, we separate production and development services by using access permissions and configuration files. Using infrastructure as code (we use Terraform) speeds up creating development and production environments. If you want to create a better development environment we recommend looking at The Twelve-Factor App [1].","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_17":{"__typename":"Paragraph","id":"e585aa5e6f5b_17","name":"5c49","type":"H4","href":null,"layout":null,"metadata":null,"text":"Tests","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_18":{"__typename":"Paragraph","id":"e585aa5e6f5b_18","name":"e55a","type":"P","href":null,"layout":null,"metadata":null,"text":"Probably the most important part of any service is the tests. Unit and integration tests provide a good overview if your services are working as expected. And for core services that have a direct impact on the company, tests are a must-have. We had great success detecting bugs in our code before releasing our code into production. The key note here is that tests should be meaningful and should test the core functionality of our service. To build upon this we use CI\u002FCD to automate the releasing and testing phases which cuts the manual labor immensely.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_19":{"__typename":"Paragraph","id":"e585aa5e6f5b_19","name":"b337","type":"H4","href":null,"layout":null,"metadata":null,"text":"Metrics","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_20":{"__typename":"Paragraph","id":"e585aa5e6f5b_20","name":"2fc1","type":"P","href":null,"layout":null,"metadata":null,"text":"While tests primarily cover our core business logic during development, we still need to monitor the behavior and health of our services during and after deployment. To do this, we create metrics that give us real-time insights into how our service performs. On top of that, we build alert policies that inform us when something is wrong. We also have automatic abort functionality in our CI\u002FCD pipeline if the metrics are above a certain threshold. But creating helpful metrics can be tricky. You can either over-monitor and overload your on-call people with alerts or under-monitor and overlook issues that are bad for the company’s performance. I recommend reading Google’s SRE book, which will help you find the middle ground [2].","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_21":{"__typename":"Paragraph","id":"e585aa5e6f5b_21","name":"b78a","type":"H4","href":null,"layout":null,"metadata":null,"text":"Simplicity","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_22":{"__typename":"Paragraph","id":"e585aa5e6f5b_22","name":"78aa","type":"P","href":null,"layout":null,"metadata":null,"text":"Adding new features is inevitable, even when the core infrastructure is intact. We need to keep in mind that making decisions that require complex systems will hurt your team in the long run. While designing your systems you need to balance complexity and costs. Usually managed cloud solutions like Pub\u002FSub are the best way to reduce complexity, but they can be more costly. Our team uses managed cloud solutions because they reduce the overall load on the engineers. We also simplify our release process. We have one rule, if you push it to main it will be tested and released. No more scripts and documents on how to release new code.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_23":{"__typename":"Paragraph","id":"e585aa5e6f5b_23","name":"f079","type":"BQ","href":null,"layout":null,"metadata":null,"text":"[1]: https:\u002F\u002F12factor.net\u002F","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_24":{"__typename":"Paragraph","id":"e585aa5e6f5b_24","name":"2b13","type":"BQ","href":null,"layout":null,"metadata":null,"text":"[2]: https:\u002F\u002Fsre.google\u002Fworkbook\u002Fmonitoring\u002F","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_25":{"__typename":"Paragraph","id":"e585aa5e6f5b_25","name":"9f33","type":"H3","href":null,"layout":null,"metadata":null,"text":"Conclusion","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"Paragraph:e585aa5e6f5b_26":{"__typename":"Paragraph","id":"e585aa5e6f5b_26","name":"e8b9","type":"P","href":null,"layout":null,"metadata":null,"text":"In this article we looked at our core data infrastructure, which is responsible for receiving and saving the analytical data from our games. Due to the large amount of data we receive, we needed to create a reliable infrastructure design that would scale with our growth. Choosing the correct technology is just one part of creating such a system. Any core system should be designed with simplicity in mind with tests, metrics, logging and development environments that give developers a safety net. We went through many iterations of our system over the last couple of years, each time learning something new.","hasDropCap":null,"dropCapImage":null,"markups":[],"codeBlockMetadata":null,"iframe":null,"mixtapeMetadata":null},"CollectionViewerEdge:collectionId:6ec626d5ad62-viewerId:lo_08ce0b10983d":{"__typename":"CollectionViewerEdge","id":"collectionId:6ec626d5ad62-viewerId:lo_08ce0b10983d","isEditor":false,"isMuting":false},"ImageMetadata:1*nTxl_7KgWQYo8PfVROUK_A.png":{"__typename":"ImageMetadata","id":"1*nTxl_7KgWQYo8PfVROUK_A.png","originalWidth":4566,"originalHeight":4566},"PostViewerEdge:postId:c5fa1610d786-viewerId:lo_08ce0b10983d":{"__typename":"PostViewerEdge","shouldIndexPostForExternalSearch":true,"id":"postId:c5fa1610d786-viewerId:lo_08ce0b10983d"},"Tag:backend":{"__typename":"Tag","id":"backend","displayTitle":"Backend","normalizedTagSlug":"backend"},"Tag:data-engineering":{"__typename":"Tag","id":"data-engineering","displayTitle":"Data Engineering","normalizedTagSlug":"data-engineering"},"Tag:google-cloud-platform":{"__typename":"Tag","id":"google-cloud-platform","displayTitle":"Google Cloud Platform","normalizedTagSlug":"google-cloud-platform"},"Tag:analytics":{"__typename":"Tag","id":"analytics","displayTitle":"Analytics","normalizedTagSlug":"analytics"},"Tag:programming":{"__typename":"Tag","id":"programming","displayTitle":"Programming","normalizedTagSlug":"programming"},"Post:c5fa1610d786":{"__typename":"Post","id":"c5fa1610d786","collection":{"__ref":"Collection:6ec626d5ad62"},"content({\"postMeteringOptions\":{}})":{"__typename":"PostContent","isLockedPreviewOnly":false,"bodyModel":{"__typename":"RichText","sections":[{"__typename":"Section","name":"fdb0","startIndex":0,"textLayout":null,"imageLayout":null,"backgroundImage":null,"videoLayout":null,"backgroundVideo":null}],"paragraphs":[{"__ref":"Paragraph:e585aa5e6f5b_0"},{"__ref":"Paragraph:e585aa5e6f5b_1"},{"__ref":"Paragraph:e585aa5e6f5b_2"},{"__ref":"Paragraph:e585aa5e6f5b_3"},{"__ref":"Paragraph:e585aa5e6f5b_4"},{"__ref":"Paragraph:e585aa5e6f5b_5"},{"__ref":"Paragraph:e585aa5e6f5b_6"},{"__ref":"Paragraph:e585aa5e6f5b_7"},{"__ref":"Paragraph:e585aa5e6f5b_8"},{"__ref":"Paragraph:e585aa5e6f5b_9"},{"__ref":"Paragraph:e585aa5e6f5b_10"},{"__ref":"Paragraph:e585aa5e6f5b_11"},{"__ref":"Paragraph:e585aa5e6f5b_12"},{"__ref":"Paragraph:e585aa5e6f5b_13"},{"__ref":"Paragraph:e585aa5e6f5b_14"},{"__ref":"Paragraph:e585aa5e6f5b_15"},{"__ref":"Paragraph:e585aa5e6f5b_16"},{"__ref":"Paragraph:e585aa5e6f5b_17"},{"__ref":"Paragraph:e585aa5e6f5b_18"},{"__ref":"Paragraph:e585aa5e6f5b_19"},{"__ref":"Paragraph:e585aa5e6f5b_20"},{"__ref":"Paragraph:e585aa5e6f5b_21"},{"__ref":"Paragraph:e585aa5e6f5b_22"},{"__ref":"Paragraph:e585aa5e6f5b_23"},{"__ref":"Paragraph:e585aa5e6f5b_24"},{"__ref":"Paragraph:e585aa5e6f5b_25"},{"__ref":"Paragraph:e585aa5e6f5b_26"}]},"validatedShareKey":"","shareKeyCreator":null},"creator":{"__ref":"User:c7d7326118b3"},"inResponseToEntityType":null,"isLocked":false,"isMarkedPaywallOnly":false,"lockedSource":"LOCKED_POST_SOURCE_NONE","mediumUrl":"https:\u002F\u002Fmedium.com\u002Foutfit7\u002Fthe-billion-events-infrastructure-c5fa1610d786","primaryTopic":{"__ref":"Topic:decb52b64abf"},"topics":[{"__typename":"Topic","slug":"programming"}],"isPublished":true,"latestPublishedVersion":"e585aa5e6f5b","visibility":"PUBLIC","postResponses":{"__typename":"PostResponses","count":0},"clapCount":20,"allowResponses":true,"isLimitedState":false,"title":"The Billion Events Infrastructure","isSeries":false,"sequence":null,"uniqueSlug":"the-billion-events-infrastructure-c5fa1610d786","socialTitle":"","socialDek":"","canonicalUrl":"","metaDescription":"","latestPublishedAt":1671007140356,"readingTime":4.990880503144655,"previewContent":{"__typename":"PreviewContent","subtitle":"Outfit7 is, at its core, a data-driven company. Collecting and analyzing performance metrics from our games holds a lot of…"},"previewImage":{"__ref":"ImageMetadata:0*YpS6May9aFFtbR0D"},"isShortform":false,"seoTitle":"","firstPublishedAt":1670941930838,"updatedAt":1671011407042,"shortformType":"SHORTFORM_TYPE_LINK","seoDescription":"","viewerEdge":{"__ref":"PostViewerEdge:postId:c5fa1610d786-viewerId:lo_08ce0b10983d"},"isSuspended":false,"license":"ALL_RIGHTS_RESERVED","tags":[{"__ref":"Tag:backend"},{"__ref":"Tag:data-engineering"},{"__ref":"Tag:google-cloud-platform"},{"__ref":"Tag:analytics"},{"__ref":"Tag:programming"}],"isNewsletter":false,"statusForCollection":"APPROVED","pendingCollection":null,"detectedLanguage":"en","wordCount":1221,"layerCake":3,"responsesLocked":false}}</script><script>window.__MIDDLEWARE_STATE__={"session":{"xsrf":""},"cache":{"cacheStatus":"MISS"}}</script><script src="https://cdn-client.medium.com/lite/static/js/manifest.b2314f6d.js"></script><script src="https://cdn-client.medium.com/lite/static/js/9865.1496d74a.js"></script><script src="https://cdn-client.medium.com/lite/static/js/main.24534aeb.js"></script><script src="https://cdn-client.medium.com/lite/static/js/instrumentation.d9108df7.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/reporting.ff22a7a5.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/9120.5df29668.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/5049.d1ead72d.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/4810.6318add7.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/6618.db187378.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/2707.b0942613.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/9977.5b3eb23a.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/8599.1ab63137.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/5250.9f9e01d2.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/6349.b071a958.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/2648.26563adf.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/8393.826a25fb.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/7079.67349d50.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/3735.afb7e926.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/5642.a2d9f6a1.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/6546.cd03f950.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/6834.08de95de.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/7346.72622eb9.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/2420.2a5e2d95.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/839.ca7937c2.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/7975.d195c6f1.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/2106.21ff89d3.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/7394.3d049572.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/2961.00a48598.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/8204.c4082863.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/4391.59acaed3.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/PostPage.MainContent.c8a11795.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/8414.6565ad5f.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/3974.8d3e0217.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/2527.a0afad8a.chunk.js"></script> <script src="https://cdn-client.medium.com/lite/static/js/PostResponsesContent.36c2ecf4.chunk.js"></script><script>window.main();</script><script>(function(){function c(){var b=a.contentDocument||a.contentWindow.document;if(b){var d=b.createElement('script');d.innerHTML="window.__CF$cv$params={r:'8e738a1f3fbece35',t:'MTczMjM4OTg2OC4wMDAwMDA='};var a=document.createElement('script');a.nonce='';a.src='/cdn-cgi/challenge-platform/scripts/jsd/main.js';document.getElementsByTagName('head')[0].appendChild(a);";b.getElementsByTagName('head')[0].appendChild(d)}}if(document.body){var a=document.createElement('iframe');a.height=1;a.width=1;a.style.position='absolute';a.style.top=0;a.style.left=0;a.style.border='none';a.style.visibility='hidden';document.body.appendChild(a);if('loading'!==document.readyState)c();else if(window.addEventListener)document.addEventListener('DOMContentLoaded',c);else{var e=document.onreadystatechange||function(){};document.onreadystatechange=function(b){e(b);'loading'!==document.readyState&&(document.onreadystatechange=e,c())}}}})();</script></body></html>