CINXE.COM

<!DOCTYPE html><html lang="en" class="__variable_403957 __variable_f8bf1a"><head><meta charSet="utf-8"/><meta name="viewport" content="width=device-width, initial-scale=1"/><link rel="preload" href="/_next/static/media/92f44bb82993d879-s.p.woff2" as="font" crossorigin="" type="font/woff2"/><link rel="preload" href="/_next/static/media/a34f9d1faa5f3315-s.p.woff2" as="font" crossorigin="" type="font/woff2"/><link rel="preload" as="image" href="/authzed-icon-multi.svg" fetchPriority="high"/><link rel="preload" as="image" imageSrcSet="/_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=640&q=75 640w, /_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=750&q=75 750w, /_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=828&q=75 828w, /_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=1080&q=75 1080w, /_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=1200&q=75 1200w, /_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=1920&q=75 1920w, /_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=2048&q=75 2048w, /_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=3840&q=75 3840w" imageSizes="100vw" fetchPriority="high"/><link rel="stylesheet" href="/_next/static/css/7bdcace9eef5fb2c.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/f54ebf99cea804b7.css" data-precedence="next"/><link rel="stylesheet" href="/_next/static/css/da72cc09e5cef6cf.css" data-precedence="next"/><link rel="preload" href="/_next/static/chunks/webpack-c6fc13eb9789d24f.js" as="script" fetchPriority="low"/><script src="/_next/static/chunks/fd9d1056-35f33f7ae5fc529e.js" async=""></script><script src="/_next/static/chunks/7864-3806b5b4c5d54493.js" async=""></script><script src="/_next/static/chunks/main-app-f98c4186aa19bb86.js" async=""></script><link rel="preload" href="https://js.hsforms.net/forms/embed/v2.js" as="script"/><link rel="preload" href="/vendor/consent-manager.js" as="script"/><link rel="preload" href="https://use.typekit.net/pjz2dde.css" as="style"/><title>Google-Scale Authorization: Getting to 1 Million QPS on SpiceDB Dedicated with CockroachDB</title><meta name="description" content="Achieve 1 million requests per second with 1% writes while maintaining 5.76ms P95 on CheckPermission. The SpiceDB Dedicated product was deployed on AWS with CockroachDB as the backing datastore."/><meta name="robots" content="index, follow, nocache"/><meta name="googlebot" content="index, follow, noimageindex, max-video-preview:-1, max-image-preview:large, max-snippet:-1"/><link rel="canonical" href="https://authzed.com/blog/google-scale-authorization"/><meta property="og:title" content="Google-Scale Authorization: Getting to 1 Million QPS on SpiceDB Dedicated with CockroachDB"/><meta property="og:description" content="Achieve 1 million requests per second with 1% writes while maintaining 5.76ms P95 on CheckPermission. The SpiceDB Dedicated product was deployed on AWS with CockroachDB as the backing datastore."/><meta property="og:image" content="https://authzed.com/images/blogs/nasa-q1p7bh3shj8-unsplash.jpg"/><meta name="twitter:card" content="summary_large_image"/><meta name="twitter:title" content="Google-Scale Authorization: Getting to 1 Million QPS on SpiceDB Dedicated with CockroachDB"/><meta name="twitter:description" content="Achieve 1 million requests per second with 1% writes while maintaining 5.76ms P95 on CheckPermission. The SpiceDB Dedicated product was deployed on AWS with CockroachDB as the backing datastore."/><meta name="twitter:image" content="https://authzed.com/images/blogs/nasa-q1p7bh3shj8-unsplash.jpg"/><meta name="next-size-adjust"/><link rel="icon" type="image/svg+xml" href="/favicon.svg"/><link rel="icon" type="image/x-icon" href="/favicon.ico"/><link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png"/><link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png"/><link rel="mask-icon" href="/safari-pinned-tab.svg" type="image/svg+xml"/><link rel="alternate" type="application/atom+xml" title="Atom Feed for authzed.com" href="/feed/atom"/><link rel="alternate" type="application/rss+xml" title="RSS Feed for authzed.com" href="/feed/rss"/><link rel="stylesheet" href="https://use.typekit.net/pjz2dde.css"/><script defer="" data-domain="authzed.com" src="/js/script.js" data-api="/api/event"></script><script src="/_next/static/chunks/polyfills-c67a75d1b6f99dc8.js" noModule=""></script></head><body class="bg-dark text-white overflow-x-hidden"><style> div[data-consent-manager-dialog] button[type=submit] { color: #fff; background-color: rgb(133, 74, 170); background-image: none; } </style><div id="consent-manager" class="fixed left-6 bottom-6 mr-6 z-[2147483647] shadow-lg [&>div]:rounded-lg [&>div]:p-4 [&>div]:pr-10 [&>div]:bg-suns-1000"></div><template data-dgst="NEXT_DYNAMIC_NO_SSR_CODE"></template><div class="bg-dark text-white"><div class="flex announcement-bar relative flex-wrap justify-between gap-y-2.5 py-3 px-5 z-50 text-[.8125rem] text-white bg-rocks-700/85 opacity-100 border-rocks-500 border-b-2"><div class="announcment grid grid-flow-col auto-cols-2 tablet:flex tablet:flex-wrap gap-2 justify-items-start items-center"><div class="flex flex-initial"><p>Office Hours <strong>Feb 27 at 9 AM PT / 12 PM ET</strong>: Solving the Dual-Write Problem - A Deep Dive</p></div><a href="https://www.youtube.com/watch?v=6lDkXrFjuhc"><button class="inline-flex items-center justify-center disabled:pointer-events-none disabled:opacity-50 focus:nonedata-[state=open]:bg-slate-100 bg-gradient-to-b from-[#FF5C61] to-[#9C3774] rounded-full border border-[#AC5184] transition-transform duration-100 hover:scale-[1.03] h-5 content-center text-[.75rem] font-medium px-3 py-2.5" label="Remind me " icon="bell">Remind me <svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="bell" class="svg-inline--fa fa-bell bg-transparent h-[0.625rem] w-[0.625rem] ml-2" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 448 512"><path fill="currentColor" d="M224 0c-17.7 0-32 14.3-32 32V51.2C119 66 64 130.6 64 208v18.8c0 47-17.3 92.4-48.5 127.6l-7.4 8.3c-8.4 9.4-10.4 22.9-5.3 34.4S19.4 416 32 416H416c12.6 0 24-7.4 29.2-18.9s3.1-25-5.3-34.4l-7.4-8.3C401.3 319.2 384 273.9 384 226.8V208c0-77.4-55-142-128-156.8V32c0-17.7-14.3-32-32-32zm45.3 493.3c12-12 18.7-28.3 18.7-45.3H224 160c0 17 6.7 33.3 18.7 45.3s28.3 18.7 45.3 18.7s33.3-6.7 45.3-18.7z"></path></svg></button></a></div><div class="hidden laptop:flex gh-container flex-wrap flex-initial self-end gap-y-2.5"><span><a href="https://github.com/authzed/spicedb" class="text-current no-underline">SpiceDB, Open Source Google Zanzibar FGA </a></span><a href="https://github.com/authzed/spicedb"><button class="w-max flex items-center content-center justify-center bg-white border-[rgba(27,31,36,.15)] font-sans text-[#323040] font-medium transition-transform duration-100 active:scale-95 hover:scale-105 focus:outline-none focus:ring-2 focus:ring-slate-400 focus:ring-offset-2 h-5 rounded-sm text-[0.6875rem] leading-[.9rem] border-[1px]" type="button"><div class="px-1 border-[1px] h-full border-r border-[rgba(27,31,36,.05)] flex flex-nowrap items-center place-content-center bg-[#EBF0F4]"><svg aria-hidden="true" focusable="false" data-prefix="fab" data-icon="github" class="svg-inline--fa fa-github fa-lg " role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 496 512"><path fill="currentColor" d="M165.9 397.4c0 2-2.3 3.6-5.2 3.6-3.3.3-5.6-1.3-5.6-3.6 0-2 2.3-3.6 5.2-3.6 3-.3 5.6 1.3 5.6 3.6zm-31.1-4.5c-.7 2 1.3 4.3 4.3 4.9 2.6 1 5.6 0 6.2-2s-1.3-4.3-4.3-5.2c-2.6-.7-5.5.3-6.2 2.3zm44.2-1.7c-2.9.7-4.9 2.6-4.6 4.9.3 2 2.9 3.3 5.9 2.6 2.9-.7 4.9-2.6 4.6-4.6-.3-1.9-3-3.2-5.9-2.9zM244.8 8C106.1 8 0 113.3 0 252c0 110.9 69.8 205.8 169.5 239.2 12.8 2.3 17.3-5.6 17.3-12.1 0-6.2-.3-40.4-.3-61.4 0 0-70 15-84.7-29.8 0 0-11.4-29.1-27.8-36.6 0 0-22.9-15.7 1.6-15.4 0 0 24.9 2 38.6 25.8 21.9 38.6 58.6 27.5 72.9 20.9 2.3-16 8.8-27.1 16-33.7-55.9-6.2-112.3-14.3-112.3-110.5 0-27.5 7.6-41.3 23.6-58.9-2.6-6.5-11.1-33.3 2.6-67.9 20.9-6.5 69 27 69 27 20-5.6 41.5-8.5 62.8-8.5s42.8 2.9 62.8 8.5c0 0 48.1-33.6 69-27 13.7 34.7 5.2 61.4 2.6 67.9 16 17.7 25.8 31.5 25.8 58.9 0 96.5-58.9 104.2-114.8 110.5 9.2 7.9 17 22.9 17 46.4 0 33.7-.3 75.4-.3 83.6 0 6.5 4.6 14.4 17.3 12.1C428.2 457.8 496 362.9 496 252 496 113.3 383.5 8 244.8 8zM97.2 352.9c-1.3 1-1 3.3.7 5.2 1.6 1.6 3.9 2.3 5.2 1 1.3-1 1-3.3-.7-5.2-1.6-1.6-3.9-2.3-5.2-1zm-10.8-8.1c-.7 1.3.3 2.9 2.3 3.9 1.6 1 3.6.7 4.3-.7.7-1.3-.3-2.9-2.3-3.9-2-.6-3.6-.3-4.3.7zm32.4 35.6c-1.6 1.3-1 4.3 1.3 6.2 2.3 2.3 5.2 2.6 6.5 1 1.3-1.3.7-4.3-1.3-6.2-2.2-2.3-5.2-2.6-6.5-1zm-11.4-14.7c-1.6 1-1.6 3.6 0 5.9 1.6 2.3 4.3 3.3 5.6 2.3 1.6-1.3 1.6-3.9 0-6.2-1.4-2.3-4-3.3-5.6-2z"></path></svg><div class="pl-1">Star</div></div><span class="py- px-1">5402</span></button></a></div></div><nav class="sticky top-0 z-100"><div class="nav-bar"><div class="left-nav"><a class="ml-4 py-2 min-w-[60px] tablet:block" href="/"><img alt="AuthZed" fetchPriority="high" width="60" height="30" decoding="async" data-nimg="1" style="color:transparent" src="/authzed-icon-multi.svg"/></a><div class="nav-links"><div class="py-6 inline-block"><a class="py-2 px-4 inline-flex items-center" href="/why-authzed">Why AuthZed?</a></div><div class="nav-link group" tabindex="0"><div class="label"><span class="mr-1">Products</span><svg class="fill-current h-4 w-4" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path d="M9.293 12.95l.707.707L15.657 8l-1.414-1.414L10 10.828 5.757 6.586 4.343 8z"></path></svg></div><div class="grid grid-cols-3 gap-2 sub-nav"><div><div class="group-header"><span class="">Managed Services</span></div><div><a tabindex="0" href="/products/authzed-dedicated">AuthZed Dedicated</a></div></div><div><div class="group-header"><span class="">Self-Hosted</span></div><div><a tabindex="0" href="/products/authzed-support">AuthZed Support</a></div><div><a tabindex="0" href="/products/spicedb-enterprise">SpiceDB Enterprise</a></div></div><div><div class="group-header"><span class="">Open Source</span></div><div><a tabindex="0" href="/spicedb">SpiceDB</a></div><div><a tabindex="0" href="/products/spicedb-operator">SpiceDB Operator</a></div><div><a tabindex="0" href="/products/spicedb-clients">SpiceDB Clients</a></div></div></div></div><div class="nav-link group" tabindex="0"><div class="label"><span class="mr-1">Resources</span><svg class="fill-current h-4 w-4" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 20 20"><path d="M9.293 12.95l.707.707L15.657 8l-1.414-1.414L10 10.828 5.757 6.586 4.343 8z"></path></svg></div><div class="grid grid-cols-4 gap-2 sub-nav"><div><div class="group-header"><span class="">Learn</span></div><div><a tabindex="0" href="/docs">Docs</a></div><div><a tabindex="0" href="https://play.authzed.com">Playground</a></div></div><div><div class="group-header"><span class="">Company</span></div><div><a tabindex="0" href="/blog">Blog</a></div><div><a tabindex="0" href="/customers">Customer Stories</a></div></div><div><div class="group-header"><span class="">Support</span></div><div><a tabindex="0" href="https://security.authzed.com">Security</a></div><div><a tabindex="0" href="https://status.authzed.com/">Status Page</a></div><div><a tabindex="0" href="/create-ticket">Submit a Ticket</a></div><div><a tabindex="0" href="/call">Schedule a Call</a></div></div><div><div class="group-header"><span class="">Community</span></div><div><a tabindex="0" href="https://authzed.com/discord">Discord</a></div><div><a tabindex="0" href="https://github.com/authzed">GitHub</a></div><div><a tabindex="0" href="https://twitter.com/authzed">Twitter</a></div><div><a tabindex="0" href="https://www.youtube.com/@authzed">YouTube</a></div><div><a tabindex="0" href="https://www.linkedin.com/company/authzed/">LinkedIn</a></div></div></div></div><div class="py-6 inline-block"><a class="py-2 px-4 inline-flex items-center" href="/pricing">Pricing</a></div><div class="py-6 inline-block"><a class="py-2 px-4 inline-flex items-center" href="/spicedb">SpiceDB</a></div><div class="py-6 inline-block"><a class="py-2 px-4 inline-flex items-center" href="/tour">Tour</a></div></div></div><div class="right-nav"><div class="nav-primary-cta"><a class="" href="/call"><div class="border-transparent rounded-[22px] border-2 w-fit" style="background:linear-gradient(to bottom, #6C3E5C, #6C3E5C) padding-box, linear-gradient(to right, #BE5D9B, #834AA9) border-box"><button class="inline-flex items-center justify-center text-sm font-medium transition-all duration-500 disabled:pointer-events-none disabled:opacity-50 focus:nonedata-[state=open]:bg-slate-100 bg-light hover:text-light hover:bg-suns-900 py-2 px-4 overflow-hidden relative h-8 rounded-[22px] button-hover-gradient bg-gradient-to-r from-[#5C344E] to-[#211E2E] text-white"><span class="z-[10] flex items-center">Schedule a Call<svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="phone-flip" class="svg-inline--fa fa-phone-flip bg-transparent h-[0.875rem] w-[0.875rem] ml-2" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path fill="currentColor" d="M347.1 24.6c7.7-18.6 28-28.5 47.4-23.2l88 24C499.9 30.2 512 46 512 64c0 247.4-200.6 448-448 448c-18 0-33.8-12.1-38.6-29.5l-24-88c-5.3-19.4 4.6-39.7 23.2-47.4l96-40c16.3-6.8 35.2-2.1 46.3 11.6L207.3 368c70.4-33.3 127.4-90.3 160.7-160.7L318.7 167c-13.7-11.2-18.4-30-11.6-46.3l40-96z"></path></svg></span></button></div></a><a class="hidden" href="/call"><div class="border-transparent rounded-[22px] border-2 w-fit" style="background:linear-gradient(to bottom, #6C3E5C, #6C3E5C) padding-box, linear-gradient(to right, #BE5D9B, #834AA9) border-box"><button class="inline-flex items-center justify-center text-sm font-medium transition-all duration-500 disabled:pointer-events-none disabled:opacity-50 focus:nonedata-[state=open]:bg-slate-100 bg-light hover:text-light hover:bg-suns-900 py-2 px-4 overflow-hidden relative h-8 rounded-[22px] button-hover-gradient bg-gradient-to-r from-[#5C344E] to-[#211E2E] text-white"><span class="z-[10] flex items-center">Schedule a Call<svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="phone-flip" class="svg-inline--fa fa-phone-flip bg-transparent h-[0.875rem] w-[0.875rem] ml-2" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path fill="currentColor" d="M347.1 24.6c7.7-18.6 28-28.5 47.4-23.2l88 24C499.9 30.2 512 46 512 64c0 247.4-200.6 448-448 448c-18 0-33.8-12.1-38.6-29.5l-24-88c-5.3-19.4 4.6-39.7 23.2-47.4l96-40c16.3-6.8 35.2-2.1 46.3 11.6L207.3 368c70.4-33.3 127.4-90.3 160.7-160.7L318.7 167c-13.7-11.2-18.4-30-11.6-46.3l40-96z"></path></svg></span></button></div></a></div></div><div class="mobile-nav-button"><button type="button" class="p-2 mr-4 inline-flex items-center text-white bg-transparent"><span class="sr-only">Open menu</span><svg class="w-6 h-6" aria-hidden="true" fill="currentColor" viewBox="0 0 20 20" xmlns="http://www.w3.org/2000/svg"><path fill-rule="evenodd" d="M3 5a1 1 0 011-1h12a1 1 0 110 2H4a1 1 0 01-1-1zM3 10a1 1 0 011-1h12a1 1 0 110 2H4a1 1 0 01-1-1zM3 15a1 1 0 011-1h12a1 1 0 110 2H4a1 1 0 01-1-1z" clip-rule="evenodd"></path></svg></button></div></div></nav><div class="mobile-nav translate-x-full"><section class="w-screen h-full right-0 absolute"><div class="p-8 pb-10 flex flex-col space-y-6 overflow-y-scroll h-full"><div class="self-end"><svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="xmark" class="svg-inline--fa fa-xmark w-[20px] h-[20px]" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 320 512"><path fill="currentColor" d="M310.6 150.6c12.5-12.5 12.5-32.8 0-45.3s-32.8-12.5-45.3 0L160 210.7 54.6 105.4c-12.5-12.5-32.8-12.5-45.3 0s-12.5 32.8 0 45.3L114.7 256 9.4 361.4c-12.5 12.5-12.5 32.8 0 45.3s32.8 12.5 45.3 0L160 301.3 265.4 406.6c12.5 12.5 32.8 12.5 45.3 0s12.5-32.8 0-45.3L205.3 256 310.6 150.6z"></path></svg></div><div class="flex flex-col"><div class="flex justify-center mb-8"><div class="inline-block"><a href="/call"><div class="border-transparent rounded-[22px] border-2 w-fit" style="background:linear-gradient(to bottom, #6C3E5C, #6C3E5C) padding-box, linear-gradient(to right, #BE5D9B, #834AA9) border-box"><button class="inline-flex items-center justify-center text-sm font-medium transition-all duration-500 disabled:pointer-events-none disabled:opacity-50 focus:nonedata-[state=open]:bg-slate-100 bg-light hover:text-light hover:bg-suns-900 py-2 px-4 overflow-hidden relative h-8 rounded-[22px] button-hover-gradient bg-gradient-to-r from-[#5C344E] to-[#211E2E] text-white"><span class="z-[10] flex items-center">Schedule a Call<svg aria-hidden="true" focusable="false" data-prefix="fas" data-icon="phone-flip" class="svg-inline--fa fa-phone-flip bg-transparent h-[0.875rem] w-[0.875rem] ml-2" role="img" xmlns="http://www.w3.org/2000/svg" viewBox="0 0 512 512"><path fill="currentColor" d="M347.1 24.6c7.7-18.6 28-28.5 47.4-23.2l88 24C499.9 30.2 512 46 512 64c0 247.4-200.6 448-448 448c-18 0-33.8-12.1-38.6-29.5l-24-88c-5.3-19.4 4.6-39.7 23.2-47.4l96-40c16.3-6.8 35.2-2.1 46.3 11.6L207.3 368c70.4-33.3 127.4-90.3 160.7-160.7L318.7 167c-13.7-11.2-18.4-30-11.6-46.3l40-96z"></path></svg></span></button></div></a></div></div><div class="pb-16"><div class="mb-10 hover:underline"><a class="py-2" href="/why-authzed">Why AuthZed?</a></div><div class="mb-10"><div class="group-title">Products</div><div class="group"><div class="group-header"><span>Managed Services</span></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="/products/authzed-dedicated">AuthZed Dedicated</a></div><div class="group-header"><span>Self-Hosted</span></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="/products/authzed-support">AuthZed Support</a></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="/products/spicedb-enterprise">SpiceDB Enterprise</a></div><div class="group-header"><span>Open Source</span></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="/spicedb">SpiceDB</a></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="/products/spicedb-operator">SpiceDB Operator</a></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="/products/spicedb-clients">SpiceDB Clients</a></div></div></div><div class="mb-10"><div class="group-title">Resources</div><div class="group"><div class="group-header"><span>Learn</span></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="/docs">Docs</a></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="https://play.authzed.com">Playground</a></div><div class="group-header"><span>Company</span></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="/blog">Blog</a></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="/customers">Customer Stories</a></div><div class="group-header"><span>Support</span></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="https://security.authzed.com">Security</a></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="https://status.authzed.com/">Status Page</a></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="/create-ticket">Submit a Ticket</a></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="/call">Schedule a Call</a></div><div class="group-header"><span>Community</span></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="https://authzed.com/discord">Discord</a></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="https://github.com/authzed">GitHub</a></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="https://twitter.com/authzed">Twitter</a></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="https://www.youtube.com/@authzed">YouTube</a></div><div><a class="py-1 block whitespace-no-wrap hover:underline" href="https://www.linkedin.com/company/authzed/">LinkedIn</a></div></div></div><div class="mb-10 hover:underline"><a class="py-2" href="/pricing">Pricing</a></div><div class="mb-10 hover:underline"><a class="py-2" href="/spicedb">SpiceDB</a></div><div class="mb-10 hover:underline"><a class="py-2" href="/tour">Tour</a></div></div></div></div></section><section class=" w-screen h-full cursor-pointer "></section></div><div><main class="w-full bg-light relative z-50 text-rocks-1000"><div class="relative flex place-content-center w-full overflow-hidden bg-cover bg-center"><div class="z-10 bg-slate-900/70 h-full w-full flex place-content-center"><div class="relative z-10 py-8 laptop:py-16 flex flex-col gap-6 w-[90%] laptop:max-w-[1080px]"><div class="hidden laptop:flex items-center flex-wrap gap-1 tablet:gap-2"><a class="flex no-underline mt-0" href="/blog/tags/engineering"><span class="tag tag-lg hover:text-light">Engineering</span></a><a class="flex no-underline mt-0" href="/blog/tags/product"><span class="tag tag-lg hover:text-light">Product</span></a></div><div><h1 class="mb-0 text-3xl tablet:text-5xl text-white">Google-Scale Authorization: Getting to 1 Million QPS on AuthZed Dedicated with CockroachDB</h1></div><div class="flex items-center flex-wrap gap-6 tablet:gap-0"><div class="flex items-center gap-3"><div class="flex"><div class="relative inline-block h-[52px] w-[52px]" style="z-index:10"><img alt="/assets/team/victor-roldan-betancort.jpg" loading="lazy" decoding="async" data-nimg="fill" class="object-fill rounded-full border-2 border-white" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;color:transparent" sizes="100vw" srcSet="/_next/image?url=%2Fassets%2Fteam%2Fvictor-roldan-betancort.jpg&w=640&q=75 640w, /_next/image?url=%2Fassets%2Fteam%2Fvictor-roldan-betancort.jpg&w=750&q=75 750w, /_next/image?url=%2Fassets%2Fteam%2Fvictor-roldan-betancort.jpg&w=828&q=75 828w, /_next/image?url=%2Fassets%2Fteam%2Fvictor-roldan-betancort.jpg&w=1080&q=75 1080w, /_next/image?url=%2Fassets%2Fteam%2Fvictor-roldan-betancort.jpg&w=1200&q=75 1200w, /_next/image?url=%2Fassets%2Fteam%2Fvictor-roldan-betancort.jpg&w=1920&q=75 1920w, /_next/image?url=%2Fassets%2Fteam%2Fvictor-roldan-betancort.jpg&w=2048&q=75 2048w, /_next/image?url=%2Fassets%2Fteam%2Fvictor-roldan-betancort.jpg&w=3840&q=75 3840w" src="/_next/image?url=%2Fassets%2Fteam%2Fvictor-roldan-betancort.jpg&w=3840&q=75"/></div></div><div class="flex flex-col gap-1"><div class="text-light"><a class="font-medium hover:text-suns-800 transition-colors duration-150 ease-in-out" href="https://www.linkedin.com/in/vroldanbet/">Victor Roldan Betancort</a></div><div class="flex text-light gap-2 font-regular text-sm"><span>Updated September 19, 2024</span><span>|</span><span>21 min read</span></div></div></div></div></div></div><img alt="" fetchPriority="high" decoding="async" data-nimg="fill" class="z-0 absolute" style="position:absolute;height:100%;width:100%;left:0;top:0;right:0;bottom:0;object-fit:cover;color:transparent;background-size:cover;background-position:50% 50%;background-repeat:no-repeat;background-image:url("data:image/svg+xml;charset=utf-8,%3Csvg xmlns='http://www.w3.org/2000/svg' %3E%3Cfilter id='b' color-interpolation-filters='sRGB'%3E%3CfeGaussianBlur stdDeviation='20'/%3E%3CfeColorMatrix values='1 0 0 0 0 0 1 0 0 0 0 0 1 0 0 0 0 0 100 -1' result='s'/%3E%3CfeFlood x='0' y='0' width='100%25' height='100%25'/%3E%3CfeComposite operator='out' in='s'/%3E%3CfeComposite in2='SourceGraphic'/%3E%3CfeGaussianBlur stdDeviation='20'/%3E%3C/filter%3E%3Cimage width='100%25' height='100%25' x='0' y='0' preserveAspectRatio='xMidYMid slice' style='filter: url(%23b);' href='data:image/gif;base64,R0lGODlhAQABAPAAAIVKqv///yH5BAAAAAAALAAAAAABAAEAAAICRAEAOw=='/%3E%3C/svg%3E")" sizes="100vw" srcSet="/_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=640&q=75 640w, /_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=750&q=75 750w, /_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=828&q=75 828w, /_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=1080&q=75 1080w, /_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=1200&q=75 1200w, /_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=1920&q=75 1920w, /_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=2048&q=75 2048w, /_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=3840&q=75 3840w" src="/_next/image?url=%2Fimages%2Fblogs%2Fnasa-q1p7bh3shj8-unsplash.jpg&w=3840&q=75"/></div><div class="w-[90%] pt-8 content-default text-rocks-500 flex flex-col laptop:max-w-[1080px] laptop:pt-16 laptop:flex-row"><div class="w-full mb-24 prose tablet:prose-tablet max-w-none leading-[2rem] laptop:w-2/3 laptop:pr-20"><article><script src="https://cdn.jsdelivr.net/npm/chart.js"></script> <h2 id="user-content-background">Background</h2> <p>We've been working hard to make our managed <a href="https://authzed.com/products/authzed-dedicated?utm_source=blog&utm_medium=website&utm_campaign=load_test">AuthZed Dedicated</a> offering capable of scaling as <a href="https://zanzibar.tech/2SuZL2BPQU:0:14" target="_blank">Google does</a>—particularly since it’s in our messaging to prospects and customers as a core reason to choose AuthZed. One of those prospects challenged us to demonstrate our bold claims with a series of load tests that would culminate with 1 million requests per second against 100 billion relationships stored.</p> <p>If you want to understand the performance of <a href="https://authzed.com/products/authzed-dedicated?utm_source=blog&utm_medium=website&utm_campaign=load_test">AuthZed Dedicated</a> configured with <a href="https://www.cockroachlabs.com/product/" target="_blank">CockroachDB</a> for data storage or are interested in running your own scale tests, you will find our experience useful. I cover our setup and methodology below, along with the results. <strong>Spoiler:</strong> 👉 we got to 1 million requests per second, with 1% writes while maintaining 5.76ms P95 on CheckPermission. 🎉</p> <div class="hs-cta-embed hs-cta-simple-placeholder hs-cta-embed-163937461080" style="max-width:100%; max-height:100%; width:621px;height:221.0416717529297px"> <a href="https://cta-service-cms2.hubspot.com/web-interactives/public/v1/track/redirect?encryptedPayload=AVxigLJytiMX3uGW4y%2BAZfiIFjRoB%2BFi4%2By4yCQ7D7JDveWF5%2B%2Bc1CeZo66q4GI0eUzWi3weBc52yFCLRI7Hi8zM9Ul1BwJbsTnptG6ucO%2BvYRhZxxYooMy1K5MxEsD0u80SPujc49I0rNNXWF95hLBJvMyOPGN%2FSclVfes9tRfLlfbaHzTaHg4%3D&webInteractiveContentId=163937461080&portalId=43591865" target="_blank" rel="noopener"> <img alt="A small request We are trying to get SpiceDB to 5k stars, can you help us out by starring the repository? It helps increase awareness of SpiceDB, our open-source, Google Zanzibar-inspired permissions database. " src="https://no-cache.hubspot.com/cta/default/43591865/interactive-163937461080.png" style="height: 100%; width: 100%; object-fit: fill"> </a> </div> <h2 id="user-content-goals">Goals</h2> <p>The goals of the test were to validate that SpiceDB exhibits similar latency metrics across a total of 13 tests with varying throughput and relationships stored, here is the full list:</p> <table><thead><tr><th>Test</th><th>Relationships Stored</th><th>* Target QPS</th></tr></thead><tbody><tr><td>T1</td><td>10,000,000</td><td>1,000</td></tr><tr><td>T2</td><td>10,000,000</td><td>10,000</td></tr><tr><td>T3</td><td>100,000,000</td><td>1,000</td></tr><tr><td>T4</td><td>100,000,000</td><td>10,000</td></tr><tr><td>T5</td><td>100,000,000</td><td>100,000</td></tr><tr><td>T6</td><td>1,000,000,000</td><td>1,000</td></tr><tr><td>T7</td><td>1,000,000,000</td><td>10,000</td></tr><tr><td>T8</td><td>1,000,000,000</td><td>100,000</td></tr><tr><td>T9</td><td>1,000,000,000</td><td>1,000,000</td></tr><tr><td>T10</td><td>100,000,000,000</td><td>1,000</td></tr><tr><td>T11</td><td>100,000,000,000</td><td>10,000</td></tr><tr><td>T12</td><td>100,000,000,000</td><td>100,000</td></tr><tr><td>T13</td><td>100,000,000,000</td><td>1,000,000</td></tr></tbody></table> <p>*<em>Includes 1% WriteRelationship, remainder are CheckPermission</em></p> <p>We had a couple of secondary goals as well:</p> <ul> <li>Collect various data points to understand and visualize the linear growth of computational resources with respect to traffic and dataset size.</li> <li>Identify how long it takes to upload data into a CockroachDB cluster. Many companies looking at a permissions database for authorization have existing data that needs migrating, and a crucial part of planning a rollout of a system like SpiceDB is understanding how long importing relationship data may take.</li> </ul> <h2 id="user-content-test-infrastructure-software-and-environment">Test Infrastructure, Software, and Environment</h2> <p>For our tests, we used our <a href="https://authzed.com/products/authzed-dedicated?utm_source=blog&utm_medium=website&utm_campaign=load_test">AuthZed Dedicated</a> product to deploy a SpiceDB Permissions System onto AWS, with <a href="https://www.cockroachlabs.com/product/" target="_blank">CockroachDB</a> as the backing datastore. AuthZed Dedicated leverages <a href="https://aws.amazon.com/eks/" target="_blank">EKS</a> to distribute pods across the available virtual machines. AuthZed Dedicated also isolates workloads onto distinct compute pools: one for our control plane, which runs supporting services like Envoy, Prometheus, Alert Manager, Grafana, etc, and another for SpiceDB itself. The earlier tests had Envoy, which manages load distribution within a SpiceDB cluster, running within each SpiceDB node, but we moved Envoy early on in our tests to its own pool within the control plane.</p> <p>In AuthZed Dedicated, SpiceDB is deployed as a cluster of pod replicas with CPU and memory resource limits. We prevent SpiceDB Clusters from scheduling multiple pods (replicas) onto a single node with anti-affinity rules, and nodes are placed across availability zones to reduce the blast radius of a failure. Additionally, our Kubernetes clusters utilize CPU pinning—SpiceDB pods have dedicated CPU cores to maximize the time a core works within the same context.</p> <p>We ended our tests with SpiceDB version 1.21.0, which included all of the improvements we implemented along the way (see: <a href="#spicedb-improvements-along-the-way">SpiceDB Improvements Along the Way</a>). For CockroachDB, we leveraged CockroachDB Dedicated, with version v22.2.11. Optimizing CockroachDB connection management is covered in <a href="https://authzed.com/blog/maximizing-cockroachdb-performance">Maximizing CockroachDB Performance: Our Journey to 1 Million QPS</a>.</p> <p>To generate test data and test load, we wrote Go programs, which we detail below.</p> <h3 id="user-content-aws-infrastructure">AWS Infrastructure</h3> <table><thead><tr><th>Test</th><th>Relationships</th><th>QPS</th><th>Control Plane</th><th>QTY</th><th>SpiceDB</th><th>QTY</th></tr></thead><tbody><tr><td>T1</td><td>10,000,000</td><td>1,000</td><td>c6a.xlarge - 4 vCPU</td><td>1</td><td>c6a.xlarge - 4 vCPU</td><td>3</td></tr><tr><td>T2</td><td>10,000,000</td><td>10,000</td><td>c6a.xlarge - 4 vCPU</td><td>1</td><td>c6a.2xlarge - 8 vCPU</td><td>4</td></tr><tr><td>T3</td><td>100,000,000</td><td>1,000</td><td>c6a.xlarge - 4 vCPU</td><td>1</td><td>c6a.xlarge - 4 vCPU</td><td>6</td></tr><tr><td>T4</td><td>100,000,000</td><td>10,000</td><td>c6a.xlarge - 4 vCPU</td><td>1</td><td>c6a.2xlarge - 8 vCPU</td><td>4</td></tr><tr><td>T5</td><td>100,000,000</td><td>100,000</td><td>c6a.xlarge - 4 vCPU</td><td>1</td><td>c6a.4xlarge - 16 vCPU</td><td>12</td></tr><tr><td>T6</td><td>1,000,000,000</td><td>1,000</td><td>c6a.xlarge - 4 vCPU</td><td>1</td><td>c6a.xlarge - 4 vCPU</td><td>3</td></tr><tr><td>T7</td><td>1,000,000,000</td><td>10,000</td><td>c6a.xlarge - 4 vCPU</td><td>2</td><td>c6a.2xlarge - 8 vCPU</td><td>3</td></tr><tr><td>T8</td><td>1,000,000,000</td><td>100,000</td><td>c6a.4xlarge - 16 vCPU</td><td>5</td><td>c6a.4xlarge - 16 vCPU</td><td>7</td></tr><tr><td>T9</td><td>1,000,000,000</td><td>1,000,000</td><td>c6a.4xlarge - 16 vCPU</td><td>36</td><td>c6a.8xlarge - 32 vCPU</td><td>25</td></tr><tr><td>T10</td><td>100,000,000,000</td><td>1,000</td><td>c6a.xlarge - 4 vCPU</td><td>4</td><td>c6a.xlarge - 4 vCPU</td><td>3</td></tr><tr><td>T11</td><td>100,000,000,000</td><td>10,000</td><td>c6a.xlarge - 4 vCPU</td><td>5</td><td>c6a.2xlarge - 8 vCPU</td><td>5</td></tr><tr><td>T12</td><td>100,000,000,000</td><td>100,000</td><td>c6a.2xlarge - 8 vCPU</td><td>9</td><td>c6a.4xlarge - 16 vCPU</td><td>7</td></tr><tr><td>T13</td><td>100,000,000,000</td><td>1,000,000</td><td>c6a.4xlarge - 16 vCPU</td><td>35</td><td>c6a.8xlarge - 32 vCPU</td><td>21</td></tr></tbody></table> <h3 id="user-content-cockroachdb-infrastructure">CockroachDB Infrastructure</h3> <table><thead><tr><th>Test</th><th>Relationships</th><th>QPS</th><th>CockroachDB</th><th>Attached Storage</th><th>QTY</th></tr></thead><tbody><tr><td>T1</td><td>10,000,000</td><td>1,000</td><td>m6i.2xlarge - 8 vCPU</td><td>75 GiB</td><td>4</td></tr><tr><td>T2</td><td>10,000,000</td><td>10,000</td><td>m6i.4xlarge - 16 vCPU</td><td>150 GiB</td><td>3</td></tr><tr><td>T3</td><td>100,000,000</td><td>1,000</td><td>m6i.4xlarge - 16 vCPU</td><td>150 GiB</td><td>3</td></tr><tr><td>T4</td><td>100,000,000</td><td>10,000</td><td>m6i.4xlarge - 16 vCPU</td><td>150 GiB</td><td>3</td></tr><tr><td>T5</td><td>100,000,000</td><td>100,000</td><td>m6i.4xlarge - 16 vCPU</td><td>150 GiB</td><td>5</td></tr><tr><td>T6</td><td>1,000,000,000</td><td>1,000</td><td>m6i.xlarge - 4 vCPU</td><td>150 GiB</td><td>3</td></tr><tr><td>T7</td><td>1,000,000,000</td><td>10,000</td><td>m6i.2xlarge - 8 vCPU</td><td>150 GiB</td><td>3</td></tr><tr><td>T8</td><td>1,000,000,000</td><td>100,000</td><td>m6i.4xlarge - 16 vCPU</td><td>150 GiB</td><td>6</td></tr><tr><td>T9</td><td>1,000,000,000</td><td>1,000,000</td><td>m6i.8xlarge - 32 vCPU</td><td>150 GiB</td><td>12</td></tr><tr><td>T10</td><td>100,000,000,000</td><td>1,000</td><td>m6i.2xlarge - 8 vCPU</td><td>2,363 GiB</td><td>6</td></tr><tr><td>T11</td><td>100,000,000,000</td><td>10,000</td><td>m6i.2xlarge - 8 vCPU</td><td>2,363 GiB</td><td>6</td></tr><tr><td>T12</td><td>100,000,000,000</td><td>100,000</td><td>m6i.4xlarge - 16 vCPU</td><td>2,363 GiB</td><td>6</td></tr><tr><td>T13</td><td>100,000,000,000</td><td>1,000,000</td><td>m6i.8xlarge - 32 vCPU</td><td>2,363 GiB</td><td>6</td></tr></tbody></table> <h2 id="user-content-slo-targets">SLO Targets</h2> <p>In addition to the target queries per second for each test, we defined success criteria. Each test was measured across a 10-minute period and needed to meet the following:</p> <ul> <li><a href="https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.PermissionsService.CheckPermission" target="_blank">CheckPermission</a> P95 of less than 25ms</li> <li><a href="https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.PermissionsService.WriteRelationships" target="_blank">WriteRelationship</a> P95 of less than 100ms</li> <li>Both needed to maintain a success rate greater than or equal to 99.9%</li> </ul> <p>We built a Grafana dashboard to capture all of the relevant metrics and manually ported them into a spreadsheet for each test. We also captured the value of various other relevant metrics to validate our assumptions on how SpiceDB behaves under load, e.g., we were able to visualize how the subproblem cache-hit rate exhibited asymptotic behavior with respect to the API QPS.</p> <h2 id="user-content-spicedb-schema-and-anticipated-load">SpiceDB Schema and Anticipated Load</h2> <p>A key part of SpiceDB is the SpiceDB Schema—it defines how entities relate to each other and the permissions those relationships drive. A SpiceDB Schema’s design drives the load on a SpiceDB cluster and the infrastructure capacity needed to maintain target SLOs. The complexity is added in the form of sub-problems that SpiceDB calculates; the <a href="https://authzed.com/blog/check-it-out-2#breaking-into-subproblems">Check it Out-2</a> blog does an excellent job of describing sub-problems in SpiceDB.</p> <p>Another consideration is your use of SpiceDB’s RPCs—our tests focused on CheckPermission and WriteRelationships from the <a href="https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.PermissionsService" target="_blank">PermissionsService</a>. If your workload requires listing resources, you will introduce the use of <a href="https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.PermissionsService.LookupResources" target="_blank">LookupResources</a>, which would impact utilization and require additional infrastructure.</p> <p>All tests were conducted against a customer-provided SpiceDB Schema, which modeled permissions for a social networking platform. Similar to how you use ORM (object-relational mapping) in an object programming language to map an object to a relational database, the concepts end-users see in a client application, e.g., a web app, need to be described in the SpiceDB Schema in a way that better serves the authorization model targeted. Here is the authorization behavior captured in the SpiceDB Schema:</p> <ul> <li>Users can share public/private content</li> <li>Users can follow each other</li> <li>Users can block other users</li> <li>Users can have conversations associated with a piece of content</li> <li>Public content can be seen by any user, private content can only be seen by followers</li> </ul> <iframe src="https://play.authzed.com/i/O0zcnXwtKDQ-/schema" width="100%" height="500" style="border:none;padding-bottom:20px;"></iframe> <p>As you can see, the schema had a healthy dose of unions, intersections, and exclusions, three levels of nesting (user, content, and content interaction), and usage of wildcards to gate visibility. Your SpiceDB Schema is likely different and introduces its own complexity, which will have an impact on infrastructure requirements. See the <a href="https://authzed.com/docs/reference/schema-lang#operations" target="_blank">SpiceDB Schema reference</a> for a full list of operators.</p> <h2 id="user-content-generating-test-data">Generating Test Data</h2> <p>We were provided with the number of users, content published, and interactions on the content as a baseline from which we derived the total number of relationships expected for each relation. Here is the breakdown:</p> <table><thead><tr><th>Relationship</th><th>1M</th><th>100M</th><th>1B</th><th>100B</th></tr></thead><tbody><tr><td>user#visibility@user:*#...</td><td>2,604</td><td>260,436</td><td>2,604,364</td><td>260,436,395</td></tr><tr><td>user#visibility@user#follower</td><td>2,604</td><td>260,436</td><td>2,604,365</td><td>260,436,495</td></tr><tr><td>user#self@user#...</td><td>5,209</td><td>520,873</td><td>5,208,729</td><td>520,872,890</td></tr><tr><td>user#follower@user#...</td><td>52,083</td><td>5,208,329</td><td>52,083,292</td><td>5,208,329,201</td></tr><tr><td>interaction#creator@user#...</td><td>104,167</td><td>10,416,658</td><td>104,166,583</td><td>10,416,658,302</td></tr><tr><td>interaction#parent@content#...</td><td>104,167</td><td>10,416,658</td><td>104,166,583</td><td>10,416,658,302</td></tr><tr><td>content#visibility@user:*#...</td><td>180,989</td><td>18,098,892</td><td>180,988,918</td><td>18,098,891,756</td></tr><tr><td>content#visibility@user#follower</td><td>183,594</td><td>18,359,412</td><td>183,594,125</td><td>18,359,412,451</td></tr><tr><td>content#owner@user#...</td><td>364,583</td><td>36,458,304</td><td>364,583,042</td><td>36,458,304,207</td></tr></tbody></table> <p>However, these numbers by themselves don’t tell us how content and interactions on that content are distributed across the users, for instance:</p> <ul> <li>How many pieces of content does the average user publish?</li> <li>How many followers does the average user have?</li> <li>What about the number of interactions by content published?</li> </ul> <h3 id="user-content-using-pareto-distribution-to-shape-the-graph">Using Pareto Distribution to Shape the Graph</h3> <p>If you think of popular social networks like Instagram, YouTube, or Twitter, it's not unreasonable to expect that some users are more active than others; some even generate content hourly and have millions of followers. We decided to follow a Pareto distribution to model the activity on a social network (see: <a href="https://www.nature.com/articles/srep01783" target="_blank">Origins of power-law degree distribution in the heterogeneity of human activity in social networks</a>). Meaning 20% of the network’s users generate 80% of the activity, comments, etc.</p> <p>Being a graph database, when testing SpiceDB it’s important to consider the shape of your data and its impact on querying and performance. For example, some users in the graph may have very wide relations (think thousands of relationships), which would require a lot of subproblem dispatching. Very nested group membership can also be expensive to compute (and that’s the reason Zanzibar includes a component named <a href="https://zanzibar.tech/21tieegnDR:0:2Y" target="_blank">Leopard Index</a> to handle this efficiently).</p> <p>We wrote a dataset generator in Go that took the number of users, pieces of content, and interactions as input, created relationships using a Pareto distribution, and outputted an <a href="https://avro.apache.org/" target="_blank">Apache Avro</a> file with the relationships represented in SQL, all compressed using <a href="https://en.wikipedia.org/wiki/Snappy_(compression)" target="_blank">Snappy</a>. The file was stored in S3 and transferred into the target CockroachDB cluster. We ran the program to generate each dataset: 10M, 100M, 1B, and 100B.</p> <p>This process kick-started our work to support bulk migration natively in the SpiceDB API. From version 1.22, SpiceDB exposes <a href="https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.ExperimentalService.BulkImportRelationships" target="_blank">bulk_import</a> and <a href="https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.ExperimentalService.BulkExportRelationships" target="_blank">bulk_export</a> RPCs that massively accelerate an initial import.</p> <h2 id="user-content-generating-load">Generating Load</h2> <p>The workload was generated by defining a script with our in-house <code>thumper</code> tool. Thumper is a domain-specific load-test tool written in Go that takes a YAML file as input. The YAML file describes the sequence of steps to execute at a specified weight to emulate the API usage patterns of your applications. We’re exploring <a href="https://github.com/authzed/spicedb/issues/1431" target="_blank">open sourcing Thumper</a>—please upvote if you’d like to see this.</p> <p>For CheckPermission, we were provided with the following distribution:</p> <ul> <li>60% of requests check if the user can see a piece of content</li> <li>30% of requests check if the user can see another user's profile</li> <li>10% of requests check if the user can see an interaction</li> </ul> <p>We also added 1% of write requests to the above distribution. 1% is more than the <a href="https://zanzibar.tech/2IudGpP-JZ:0:1O" target="_blank">0.19% Google’s Zanzibar paper documented as peak writes</a>, but a lot of their usage comes from <a href="https://zanzibar.tech/25xSoW13un:0.V3atsH9If:0" target="_blank">Read</a> requests, suggesting Zanzibar is doing more than just authorization at Google.</p> <p>The load-test script issued <code>CheckPermission</code> with <code>consistency</code> of <code>minimize_latency</code>, and the quantization window remained at the default 5 seconds, with a max staleness of 100% (another 5s seconds of potential staleness). If you want to know more about SpiceDB caching strategies, check out Jake's blog post <a href="https://authzed.com/blog/hotspot-caching-in-google-zanzibar-and-spicedb">Hotspot Caching in Google Zanzibar and SpiceDB</a>. Jake’s post was inspired by one of the few performance bottlenecks we fixed as we started cranking up the QPS knob to some serious numbers.</p> <h3 id="user-content-modeling-daus">Modeling DAUs</h3> <p>The numbers above by themselves are insufficient as well, e.g., which users, content, and interactions would actually make the requests? To model a real-world scenario we created synthetic Daily Active Users (DAUs) with a <strong>sampling factor</strong>. The sampling factor instructed the load-test generator to loop over a uniformly distributed sample of the user pool. For example, if we had 1,000 users and a sampling factor of 1%, the tool creates a pool of users with IDs 0, 100, 200... until user 1000, and then randomly samples from this pool of users.</p> <p>The sampling factor is a key setting in a load test, as it has a direct impact on the volume of relationships SpiceDB needs to load from the datastore, and informs the compute required to withstand a specific workload. After all, Zanzibar has a <a href="https://zanzibar.tech/23iStt3hOC:0:40" target="_blank">strong focus on minimizing database access</a> as much as possible, and many of its core design choices are at the service of sparing every last bit of database capacity.</p> <h2 id="user-content-results">Results</h2> <p>We succeeded at not only scaling to 1M QPS and 100B relationships but achieved consistent performance across all tests. 🎉 Along the way we identified and implemented performance improvements to SpiceDB (detailed in <a href="#spicedb-improvements-along-the-way">SpiceDB Improvements Along the Way</a>. Somethings to keep in mind with our results:</p> <ul> <li>T1 through T5 had Envoy running on each SpiceDB node, we moved Envoy into an isolated control plane node pool after T5.</li> <li>T9 introduced crossfade revisions (<a href="https://github.com/authzed/spicedb/pull/1285" target="_blank">PR #1285</a>), which significantly improved performance.</li> <li>Tests T10 through T13 were conducted using version 1.21.0, which incorporated all of the performance improvements we uncovered in the tests before it.</li> </ul> <p>Earlier tests were not revisited as we improved SpiceDB and AuthZed Dedicated since the updates were implemented to achieve the scale in our final test of 1M QPS and 100B relationships stored.</p> <h3 id="user-content-1000-qps">1,000 QPS</h3> <h4 id="user-content-checkpermission-latency">CheckPermission Latency</h4> <table><thead><tr><th>Test</th><th>Relationships</th><th>Cache Hit</th><th>P95 Check Latency</th><th>P50 Check Latency</th></tr></thead><tbody><tr><td>T1</td><td>10,000,000</td><td>33.20%</td><td>19.7</td><td>5.39</td></tr><tr><td>T3</td><td>100,000,000</td><td>5.00%</td><td>23.2</td><td>7.03</td></tr><tr><td>T6</td><td>1,000,000,000</td><td>21.60%</td><td>18</td><td>37</td></tr><tr><td>T10</td><td>100,000,000,000</td><td>0.00%</td><td>17.1</td><td>8.05</td></tr></tbody></table> <h4 id="user-content-writerelationship-latency">WriteRelationship Latency</h4> <table><thead><tr><th>Test</th><th>Relationships</th><th>Cache Hit</th><th>P95 Write Latency</th><th>P50 Write Latency</th></tr></thead><tbody><tr><td>T1</td><td>10,000,000</td><td>33.20%</td><td>30</td><td>14.50</td></tr><tr><td>T3</td><td>100,000,000</td><td>5.00%</td><td>20.9</td><td>13.00</td></tr><tr><td>T6</td><td>1,000,000,000</td><td>21.60%</td><td>53</td><td>16.80</td></tr><tr><td>T10</td><td>100,000,000,000</td><td>0.00%</td><td>17.9</td><td>12.90</td></tr></tbody></table> <h3 id="user-content-10000-qps">10,000 QPS</h3> <h4 id="user-content-checkpermission-latency-1">CheckPermission Latency</h4> <table><thead><tr><th>Test</th><th>Relationships</th><th>Cache Hit</th><th>P95 Check Latency</th><th>P50 Check Latency</th></tr></thead><tbody><tr><td>T2</td><td>10,000,000</td><td>72.40%</td><td>16.3</td><td>3.49</td></tr><tr><td>T4</td><td>100,000,000</td><td>72.40%</td><td>14.8</td><td>3.43</td></tr><tr><td>T7</td><td>1,000,000,000</td><td>73.70%</td><td>8.67</td><td>3.28</td></tr><tr><td>T11</td><td>100,000,000,000</td><td>65.10%</td><td>10</td><td>3.44</td></tr></tbody></table> <h4 id="user-content-writerelationship-latency-1">WriteRelationship Latency</h4> <table><thead><tr><th>Test</th><th>Relationships</th><th>Cache Hit</th><th>P95 Write Latency</th><th>P50 Write Latency</th></tr></thead><tbody><tr><td>T2</td><td>10,000,000</td><td>72.40%</td><td>22.8</td><td>13.20</td></tr><tr><td>T4</td><td>100,000,000</td><td>72.40%</td><td>22.8</td><td>12.10</td></tr><tr><td>T7</td><td>1,000,000,000</td><td>73.70%</td><td>32.9</td><td>13.80</td></tr><tr><td>T11</td><td>100,000,000,000</td><td>65.10%</td><td>37</td><td>14.30</td></tr></tbody></table> <h3 id="user-content-100000-qps">100,000 QPS</h3> <h4 id="user-content-checkpermission-latency-2">CheckPermission Latency</h4> <table><thead><tr><th>Test</th><th>Relationships</th><th>Cache Hit</th><th>P95 Check Latency</th><th>P50 Check Latency</th></tr></thead><tbody><tr><td>T5</td><td>100,000,000</td><td>87.60%</td><td>11</td><td>3.21</td></tr><tr><td>T8</td><td>1,000,000,000</td><td>89.30%</td><td>5.93</td><td>3.12</td></tr><tr><td>T12</td><td>100,000,000,000</td><td>89.20%</td><td>5.86</td><td>3.08</td></tr></tbody></table> <h4 id="user-content-writerelationship-latency-2">WriteRelationship Latency</h4> <table><thead><tr><th>Test</th><th>Relationships</th><th>Cache Hit</th><th>P95 Write Latency</th><th>P50 Write Latency</th></tr></thead><tbody><tr><td>T5</td><td>100,000,000</td><td>87.60%</td><td>56.6</td><td>17.70</td></tr><tr><td>T8</td><td>1,000,000,000</td><td>89.30%</td><td>42</td><td>15.60</td></tr><tr><td>T12</td><td>100,000,000,000</td><td>89.20%</td><td>49.3</td><td>14.90</td></tr></tbody></table> <h3 id="user-content-1000000-qps">1,000,000 QPS</h3> <h4 id="user-content-checkpermission-latency-3">CheckPermission Latency</h4> <table><thead><tr><th>Test</th><th>Relationships</th><th>Cache Hit</th><th>P95 Check Latency</th><th>P50 Check Latency</th></tr></thead><tbody><tr><td>T9</td><td>1,000,000,000</td><td>95.70%</td><td>5.89</td><td>3.1</td></tr><tr><td>T13</td><td>100,000,000,000</td><td>95.90%</td><td>5.76</td><td>3.03</td></tr></tbody></table> <h4 id="user-content-writerelationship-latency-3">WriteRelationship Latency</h4> <table><thead><tr><th>Test</th><th>Relationships</th><th>Cache Hit</th><th>P95 Write Latency</th><th>P50 Write Latency</th></tr></thead><tbody><tr><td>T9</td><td>1,000,000,000</td><td>95.70%</td><td>74.1</td><td>16.10</td></tr><tr><td>T13</td><td>100,000,000,000</td><td>95.90%</td><td>48.3</td><td>15.80</td></tr></tbody></table> <p>We also collected data on how long it takes to import a large amount of relationships into CockroachDB—which seems very reasonable. The largest import of 100B relationships took just over 5 days, but we think this can be lowered by making sure the dataset is split evenly across smaller files. The import was achieved using the AVRO file format and Snappy compression.</p> <table><thead><tr><th>Relationships</th><th>*Total Bytes per Relationship</th><th>**Total Relationship Storage</th><th>Import Time</th></tr></thead><tbody><tr><td>10,000,000</td><td>15</td><td>0.39 GiB</td><td>45s</td></tr><tr><td>100,000,000</td><td>29</td><td>7.85 GiB</td><td>6m 20s</td></tr><tr><td>1,000,000,000</td><td>42</td><td>116.48 GiB</td><td>2h 40m</td></tr><tr><td>100,000,000,000</td><td>29</td><td>7,948.15 GIB</td><td>121h 24m</td></tr></tbody></table> <p><em>* Includes 3x replication factor, *</em> Doesn’t include other CockroachDB system related storage*</p> <p>The graph below shows P95 CheckPermission and WriteRelationship latency, along with the observed Sub-Problem Cache Hit ratio, observed in the 4 tests conducted against 100B relationships stored: 1K, 10K, 100K, and 1M QPS.</p> <div class="relative" style="min-height: 320px"><canvas id="user-content-myChart"></canvas></div> <script defer> var ctx = document.getElementById('user-content-myChart').getContext('2d'); var myChart = new Chart(ctx, { type: 'line', data: { labels: ['T10', 'T11', 'T12', 'T13'], datasets: [{ label: 'P95 CheckPermission Latency', data: [17.1,10,5.86,5.76], yAxisID: 'y', borderColor: '#E1769E', fill: false, pointBackgroundColor: '#E1769E' },{ label: 'P95 WritesRelationship Latency', data: [17.9,37,49.3,48.3], yAxisID: 'y', borderColor: '#E9B674', fill: false, pointBackgroundColor: '#E9B674' },{ label: 'Sub Problem Cache Hit Percentage', data: [0,.651,.892,.959], yAxisID: 'y1', borderColor: '#A7A2B3', fill: false, pointBackgroundColor: '#A7A2B3' }] }, options: { maintainAspectRatio: false, scales: { y: { beginAtZero: true, title: { display: true, text: 'Latency (ms)' } }, y1: { beginAtZero: true, position: 'right', title: { display: true, text: 'Sub Problem Cache Hit Rate' } }, x: { title: { display: true, text: 'Test' } } }, plugins: { title: { display: true, text: "P95 Check and Write Latency Against 100B Relationships", padding: { top: 10, bottom: 30 } }, legend: { position: 'bottom', labels: { color: '#07060a', boxWidth: 10, padding: 20, font: { size: 12, }, usePointStyle: true } } } } }); </script> <h3 id="user-content-spicedb-improvements-along-the-way">SpiceDB Improvements Along the Way</h3> <p>While working through the tests, we uncovered areas for improvement and implemented them as we proceeded. The list of improvements are:</p> <ul> <li>CockroachDB connection balancing and pruning to reduce connection establishment overhead and better balance load across CockroachDB nodes (see <a href="https://authzed.com/blog/maximizing-cockroachdb-performance">Maximizing CockroachDB Performance: Our Journey to 1 Million QPS</a>)</li> <li>Quantization windows smearing to prevent connection-pool acquisition thundering herds at the end of a quantization window (see <a href="https://authzed.com/blog/hotspot-caching-in-google-zanzibar-and-spicedb">Hotspot Caching in Google Zanzibar and SpiceDB</a>)</li> <li>Fix to the hashring getting recomputed any time a sub-connection transitioned from ready to idle and vice-versa, which happened very frequently - <a href="https://github.com/authzed/spicedb/pull/1310" target="_blank">PR #1310</a></li> <li>Better balancing of dispatch operations in the cluster by increasing the replication factor, which meant all nodes received an equal amount of dispatch calls</li> <li>Use static-CPU management in Kubernetes nodes</li> </ul> <h3 id="user-content-additional-findings">Additional Findings</h3> <ul> <li>Generating 100B relationships took 24 hours on a single thread.</li> <li>Generated AVRO files were 700GB.</li> <li>Once imported and indexed, our largest dataset of 100B relationships used a total of 8TB (this includes a replication factor of 3).</li> <li>⭐ We reran T13 (not captured in the data above) with <a href="https://aws.amazon.com/ec2/graviton/" target="_blank">Amazon’s ARM Graviton-based EC2 instances</a> and observed 20% more throughput.</li> </ul> <h2 id="user-content-lessons-learned">Lessons Learned</h2> <p>A key thing to note is the interdependency between variables in the test and their impact on each other, e.g., an increase in the cache ratio decreases SpiceDB’s CPU % and CockroachDB’s QPS. Here is a partial causal loop diagram depicting the behavior we witnessed during the tests:</p> <p><img src="/images/causal-diagram.png" alt="Causal loop diagram showing the interdependency of components in SpiceDB." title="SpiceDB Component Interdependency "></p> <p>We also learned a lot about planning and executing large-scale load tests against CockroachDB. Here are some of our learnings:</p> <ul> <li>If you are importing from S3 into CockroachDB, make sure the estimated import time does not exceed the expiration of an S3 pre-signed URL.</li> <li>Disable incremental backups for the load test, as the first backup could take abnormally long and ruin the load-test session.</li> <li>A CockroachDB Dedicated cluster can be scaled down to save costs while tests are not running. The duration is proportional to the number of nodes, as they have to be drained one by one and spun back up.</li> <li>Scaling in or out a cluster with this amount of data can take significant time and balloon costs, as data needs to be replicated and shards get rebalanced.</li> <li>You may need to get granular with your metrics to discover issues, e.g., we didn’t notice spikes in connection acquisition until we instructed Prometheus to scrape at 1-second intervals instead of the 30-second we were using.</li> <li>Default Spot-Instance limits can be easily hit at this scale, and with high machine count and market volatility, you may lose those savings if a test needs to be restarted.</li> </ul> <p>If you have any questions about the test - don’t hesitate to reach out on our <a href="https://authzed.com/discord" target="_blank">Discord</a>! I’m <strong>@vroldanbet.</strong></p> <h2 id="user-content-additional-reading">Additional Reading</h2> <ul> <li><a href="https://authzed.com/blog/what-is-google-zanzibar">Understanding Google Zanzibar: A Comprehensive Overview</a></li> <li><a href="https://authzed.com/zanzibar">Google's Zanzibar White Paper, Annotated by AuthZed</a></li> <li><a href="https://authzed.com/blog/hotspot-caching-in-google-zanzibar-and-spicedb">Hotspot Caching in Google Zanzibar and SpiceDB</a></li> <li><a href="https://authzed.com/blog/maximizing-cockroachdb-performance">Maximizing CockroachDB Performance: Our Journey to 1 Million QPS</a></li> <li><a href="https://authzed.com/blog/check-it-out-2#breaking-into-subproblems">Check it Out-2: Breaking Into Subproblems</a></li> <li><a href="https://authzed.com/products/authzed-dedicated?utm_source=blog&utm_medium=website&utm_campaign=load_test">AuthZed Dedicated</a></li> <li><a href="https://authzed.com/blog/authz-primer">A Primer on Modern Enterprise Authorization (AuthZ) Systems</a></li> <li><a href="https://authzed.com/blog/fine-grained-access-control">Fine-Grained Access Control: Can You Go Too Fine?</a></li> <li><a href="https://authzed.com/blog/exploring-rebac">Relationship Based Access Control (ReBAC): Using Graphs to Power your Authorization System</a></li> <li><a href="https://authzed.com/blog/pitfalls-of-jwt-authorization">Pitfalls of JWT Authorization</a></li> </ul></article><div class="mb-6 p-4 bg-rocks-100/70 rounded text-sm italic leading-relaxed">Originally published July 12, 2023</div></div><aside class="w-1/3 px-6 mb-6 text-rocks-1000 border-l sticky top-[80px] self-start ml-[1.25rem] max-h-[100-vh] overflow-y-auto"><div class="hidden laptop:block"><div class="text-body-sm font-bold mb-4">Table of Contents</div><div class="flex flex-col gap-2 max-w-[300px]"><div><div class="pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#background">Background</a></div></div><div><div class="pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#goals">Goals</a></div></div><div><div class="pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#test-infrastructure-software-and-environment">Test Infrastructure, Software, and Environment</a></div></div><div><div class="pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#aws-infrastructure">AWS Infrastructure</a></div></div><div><div class="pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#cockroachdb-infrastructure">CockroachDB Infrastructure</a></div></div><div><div class="pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#slo-targets">SLO Targets</a></div></div><div><div class="pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#spicedb-schema-and-anticipated-load">SpiceDB Schema and Anticipated Load</a></div></div><div><div class="pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#generating-test-data">Generating Test Data</a></div></div><div><div class="pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#using-pareto-distribution-to-shape-the-graph">Using Pareto Distribution to Shape the Graph</a></div></div><div><div class="pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#generating-load">Generating Load</a></div></div><div><div class="pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#modeling-daus">Modeling DAUs</a></div></div><div><div class="pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#results">Results</a></div></div><div><div class="pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#1000-qps">1,000 QPS</a></div></div><div><div class="pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#checkpermission-latency">CheckPermission Latency</a></div></div><div><div class="pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#writerelationship-latency">WriteRelationship Latency</a></div></div><div><div class="pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#10000-qps">10,000 QPS</a></div></div><div><div class="pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#checkpermission-latency-1">CheckPermission Latency</a></div></div><div><div class="pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#writerelationship-latency-1">WriteRelationship Latency</a></div></div><div><div class="pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#100000-qps">100,000 QPS</a></div></div><div><div class="pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#checkpermission-latency-2">CheckPermission Latency</a></div></div><div><div class="pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#writerelationship-latency-2">WriteRelationship Latency</a></div></div><div><div class="pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#1000000-qps">1,000,000 QPS</a></div></div><div><div class="pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#checkpermission-latency-3">CheckPermission Latency</a></div></div><div><div class="pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#writerelationship-latency-3">WriteRelationship Latency</a></div></div><div><div class="pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#spicedb-improvements-along-the-way">SpiceDB Improvements Along the Way</a></div></div><div><div class="pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#additional-findings">Additional Findings</a></div></div><div><div class="pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#lessons-learned">Lessons Learned</a></div></div><div><div class="pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800"><div>•</div><a href="#additional-reading">Additional Reading</a></div></div></div><div class="mt-8"><div class="text-md font-bold">Subscribe to Our Newsletter</div><div class="text-sm">Stay updated on new releases, features, and announcements.</div><div class="mt-2"><div id="blog-hs-form" class="[&_form]:flex [&_form]:flex-col [&_form]:gap-4 [&_.hs-submit]:text-right [&_.hs-button]:px-4 [&_.hs-button]:py-2 [&_.hs-button]:border [&_.hs-button]:border-suns-1000 [&_.hs-button]:bg-suns-1000 [&_.hs-button]:hover:bg-suns-1000/80 [&_.hs-button]:hover:cursor-pointer [&_.hs-button]:transition-all [&_.hs-button]:rounded-lg [&_.hs-button]:text-white [&_.hs-button]:font-medium [&_.input]:!m-0 [&_.hs-firstname_.input]:!mr-2 [&_.hs-input]:!w-full [&_.hs-input]:mt-1 [&_.hs-input]:p-2 [&_.hs-input]:text-rocks-700 [&_.hs-input]:bg-rocks-100 [&_.hs-input]:border [&_.hs-input]:border-rocks-300 [&_.hs-input]:rounded-lg [&_.hs-form-required]:ml-1 [&_.hs-error-msgs]:mt-1 [&_.hs-error-msg]:text-heat-600 [&_.hs-error-msg]:text-sm">...</div></div></div></div></aside></div><div class="w-[90%] content-default laptop:max-w-[1080px]"><div class="p-8 tablet:p-16 w-full flex flex-col gap-5 text-center text-white rounded-lg border border-[#C87CC9] bg-gradient-to-r from-[#AB58AA] to-[#E1769E]"><h3 class="font-new-hero text-2xl tablet:text-4xl font-semibold">See AuthZed in action</h3><div class="text-md tablet:text-lg mb-4">Build delightful, secure application experiences with AuthZed.</div><div class="flex justify-center self-center gap-4"><a href="/call"><button class="inline-flex items-center justify-center transition-all duration-500 disabled:pointer-events-none disabled:opacity-50 focus:nonedata-[state=open]:bg-slate-100 border h-14 text-[1.0625rem] font-semibold px-4 py-3 tablet:px-8 tablet:py-5 text-rocks-500 hover:text-white bg-white hover:bg-suns-900 border-rocks-100 hover:border-suns-800 rounded-md" label="Book demo">Book demo</button></a></div></div></div></main><div class="w-full content-section bg-light"><section class="pt-7 pb-14 md:py-20 content-default"><div class="w-full pb-14 border-t border-rocks-100"></div><div class="self-center w-full"><div class="nav-links footer footer-light"><div class="group"><div class="group-title">Products</div><div><div class="group-header"><span>Managed Services</span></div><div><a href="/products/authzed-dedicated">AuthZed Dedicated</a></div><div class="group-header"><span>Self-Hosted</span></div><div><a href="/products/authzed-support">AuthZed Support</a></div><div><a href="/products/spicedb-enterprise">SpiceDB Enterprise</a></div><div class="group-header"><span>Open Source</span></div><div><a href="/spicedb">SpiceDB</a></div><div><a href="/products/spicedb-operator">SpiceDB Operator</a></div><div><a href="/products/spicedb-clients">SpiceDB Clients</a></div></div></div><div class="group"><div class="group-title">Resources</div><div><div class="group-header"><span>Learn</span></div><div><a href="https://play.authzed.com">Playground</a></div><div><a href="/docs">Docs</a></div><div><a href="/tour">Tour</a></div><div><a href="/customers">Customer Stories</a></div><div><a href="/blog/what-is-google-zanzibar">Learn More About Google Zanzibar</a></div><div class="group-header"><span>Support</span></div><div><a href="https://security.authzed.com">Security</a></div><div><a href="https://status.authzed.com/">Status Page</a></div><div><a href="/create-ticket">Submit a Ticket</a></div><div><a href="/call">Schedule a Call</a></div><div><a href="https://app.authzed.com">Serverless Login</a></div></div></div><div class="group"><div class="group-title">Company</div><div><div class="group-header"><span>Info</span></div><div><a href="/about">About Us</a></div><div><a href="/blog">Blog</a></div><div><a href="/contact-us">Contact</a></div><div><a href="https://www.workatastartup.com/companies/authzed">Join the Team</a></div><div class="group-header"><span>Legal</span></div><div><a href="/privacy-policy">Privacy Policy</a></div><div><a href="/terms-conditions">Terms & Conditions</a></div></div></div><div class="group"><div class="group-title">Community</div><div><div><a href="https://authzed.com/discord">Discord</a></div><div><a href="https://github.com/authzed">GitHub</a></div><div><a href="https://twitter.com/authzed">Twitter</a></div><div><a href="https://www.youtube.com/channel/UCFeSgZf0rPqQteiTQNGgTPg">YouTube</a></div><div><a href="https://www.linkedin.com/company/authzed/">LinkedIn</a></div></div></div></div></div><div class="my-12 px-6 flex flex-col tablet:flex-row gap-16 tablet:gap-0"><div class="justify-self-center"><a href="/"><img alt="AuthZed" loading="lazy" width="142" height="38.51" decoding="async" data-nimg="1" style="color:transparent" src="/authzed-logo-multi-dark.svg"/></a></div><div class="tablet:ml-auto flex gap-4"><img alt="Cloud Native Computing Foundation" loading="lazy" width="170" height="48" decoding="async" data-nimg="1" class="w-[170px] h-[48px] tablet:w-[170px] tablet:h-[48px]" style="color:transparent" src="/assets/logo-cncf-color.svg"/><img alt="SOC for Service Organizations" loading="lazy" width="170" height="168" decoding="async" data-nimg="1" class="w-[55px] h-[55px] tablet:w-[75px] tablet:h-[75px] self-end" style="color:transparent" src="/assets/SOC_NonCPA.svg"/></div></div></section></div></div></div><template data-dgst="NEXT_DYNAMIC_NO_SSR_CODE"></template><div></div><script src="/_next/static/chunks/webpack-c6fc13eb9789d24f.js" async=""></script><script>(self.__next_f=self.__next_f||[]).push([0])</script><script>self.__next_f.push([1,"1:HL[\"/_next/static/media/92f44bb82993d879-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n2:HL[\"/_next/static/media/a34f9d1faa5f3315-s.p.woff2\",\"font\",{\"crossOrigin\":\"\",\"type\":\"font/woff2\"}]\n3:HL[\"/_next/static/css/7bdcace9eef5fb2c.css\",\"style\"]\n0:\"$L4\"\n"])</script><script>self.__next_f.push([1,"5:HL[\"/_next/static/css/f54ebf99cea804b7.css\",\"style\"]\n6:HL[\"/_next/static/css/da72cc09e5cef6cf.css\",\"style\"]\n"])</script><script>self.__next_f.push([1,"7:I{\"id\":6054,\"chunks\":[\"2272:static/chunks/webpack-c6fc13eb9789d24f.js\",\"2971:static/chunks/fd9d1056-35f33f7ae5fc529e.js\",\"7864:static/chunks/7864-3806b5b4c5d54493.js\"],\"name\":\"\",\"async\":false}\n9:I{\"id\":1729,\"chunks\":[\"2272:static/chunks/webpack-c6fc13eb9789d24f.js\",\"2971:static/chunks/fd9d1056-35f33f7ae5fc529e.js\",\"7864:static/chunks/7864-3806b5b4c5d54493.js\"],\"name\":\"\",\"async\":false}\na:I{\"id\":8032,\"chunks\":[\"5878:static/chunks/9da6db1e-c7f8db75983a0531.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8"])</script><script>self.__next_f.push([1,"475:static/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"7967:static/chunks/7967-ff831c2c55ce81b3.js\",\"3185:static/chunks/app/layout-861f2c7fab29b964.js\"],\"name\":\"PHProvider\",\"async\":false}\nb:\"$Sreact.suspense\"\nc:I{\"id\":3388,\"chunks\":[\"3676:static/chunks/870fdd6f-9c571350bc2478c3.js\",\"3958:static/chunks/69b09407-0b4f7c53994b856f.js\",\"4724:static/chunks/4724-69a117ad21aeeee7.js\",\"3718:static/chunks/3718-0050314636e810ad.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:s"])</script><script>self.__next_f.push([1,"tatic/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"1279:static/chunks/1279-23ffe5d7b074e3d8.js\",\"3959:static/chunks/app/(main)/blog/[slug]/page-0b21ba72e4a8ca90.js\"],\"name\":\"NoSSR\",\"async\":false}\nd:I{\"id\":9243,\"chunks\":[\"5878:static/chunks/9da6db1e-c7f8db75983a0531.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:static/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"7967:static/chunks/7967-ff831c2c55ce81b3.js\",\"3185:static/chunks/app/lay"])</script><script>self.__next_f.push([1,"out-861f2c7fab29b964.js\"],\"name\":\"\",\"async\":false}\n"])</script><script>self.__next_f.push([1,"e:I{\"id\":3012,\"chunks\":[\"3676:static/chunks/870fdd6f-9c571350bc2478c3.js\",\"3958:static/chunks/69b09407-0b4f7c53994b856f.js\",\"6990:static/chunks/13b76428-98e8ebe465d89db3.js\",\"4724:static/chunks/4724-69a117ad21aeeee7.js\",\"3718:static/chunks/3718-0050314636e810ad.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:static/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"1279:static/chunks/1279-23ffe5d7b074e3d8.js\",\"7967:static/chunks/7967-ff831c2c55ce81b3.js\",\"5978:static/chunks/5978-044c34f5a716c8b6.js\",\"6377:static/chunks/6377-2505d413b8312f37.js\",\"9871:static/chunks/9871-24e0e967362f76fb.js\",\"3266:static/chunks/3266-c80e45b9d5385976.js\",\"5123:static/chunks/5123-e529ddc3b876db62.js\",\"1454:static/chunks/1454-943dee969c73969b.js\",\"7744:static/chunks/7744-6aaf36a132e6228e.js\",\"7974:static/chunks/app/(main)/page-cfcfdd7d130c8d8b.js\"],\"name\":\"SegmentProvider\",\"async\":false}\n"])</script><script>self.__next_f.push([1,"f:I{\"id\":1443,\"chunks\":[\"2272:static/chunks/webpack-c6fc13eb9789d24f.js\",\"2971:static/chunks/fd9d1056-35f33f7ae5fc529e.js\",\"7864:static/chunks/7864-3806b5b4c5d54493.js\"],\"name\":\"\",\"async\":false}\n10:I{\"id\":8639,\"chunks\":[\"2272:static/chunks/webpack-c6fc13eb9789d24f.js\",\"2971:static/chunks/fd9d1056-35f33f7ae5fc529e.js\",\"7864:static/chunks/7864-3806b5b4c5d54493.js\"],\"name\":\"\",\"async\":false}\n11:I{\"id\":4724,\"chunks\":[\"3676:static/chunks/870fdd6f-9c571350bc2478c3.js\",\"3958:static/chunks/69b09407-0b4f7c53994b856f."])</script><script>self.__next_f.push([1,"js\",\"4724:static/chunks/4724-69a117ad21aeeee7.js\",\"3718:static/chunks/3718-0050314636e810ad.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:static/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"1279:static/chunks/1279-23ffe5d7b074e3d8.js\",\"3959:static/chunks/app/(main)/blog/[slug]/page-0b21ba72e4a8ca90.js\"],\"name\":\"\",\"async\":false}\n12:I{\"id\":5827,\"chunks\":[\"3676:static/chunks/870fdd6f-9c571350bc2478c3.js\",\"3958:static/chunks/69b09407-0b4f7c53994b856f.js\",\"4724:static/c"])</script><script>self.__next_f.push([1,"hunks/4724-69a117ad21aeeee7.js\",\"3718:static/chunks/3718-0050314636e810ad.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:static/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"1279:static/chunks/1279-23ffe5d7b074e3d8.js\",\"4095:static/chunks/app/(main)/layout-997a57fd753f58b1.js\"],\"name\":\"\",\"async\":false}\n13:I{\"id\":6987,\"chunks\":[\"3676:static/chunks/870fdd6f-9c571350bc2478c3.js\",\"3958:static/chunks/69b09407-0b4f7c53994b856f.js\",\"4724:static/chunks/4724-69a117ad21aeeee7."])</script><script>self.__next_f.push([1,"js\",\"3718:static/chunks/3718-0050314636e810ad.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:static/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"1279:static/chunks/1279-23ffe5d7b074e3d8.js\",\"4095:static/chunks/app/(main)/layout-997a57fd753f58b1.js\"],\"name\":\"\",\"async\":false}\n"])</script><script>self.__next_f.push([1,"16:I{\"id\":3012,\"chunks\":[\"3676:static/chunks/870fdd6f-9c571350bc2478c3.js\",\"3958:static/chunks/69b09407-0b4f7c53994b856f.js\",\"6990:static/chunks/13b76428-98e8ebe465d89db3.js\",\"4724:static/chunks/4724-69a117ad21aeeee7.js\",\"3718:static/chunks/3718-0050314636e810ad.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:static/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"1279:static/chunks/1279-23ffe5d7b074e3d8.js\",\"7967:static/chunks/7967-ff831c2c55ce81b3.js\",\"5978:static/chunks/5978-044c34f5a716c8b6.js\",\"6377:static/chunks/6377-2505d413b8312f37.js\",\"9871:static/chunks/9871-24e0e967362f76fb.js\",\"3266:static/chunks/3266-c80e45b9d5385976.js\",\"5123:static/chunks/5123-e529ddc3b876db62.js\",\"1454:static/chunks/1454-943dee969c73969b.js\",\"7744:static/chunks/7744-6aaf36a132e6228e.js\",\"7974:static/chunks/app/(main)/page-cfcfdd7d130c8d8b.js\"],\"name\":\"SegmentPageView\",\"async\":false}\n"])</script><script>self.__next_f.push([1,"17:I{\"id\":4400,\"chunks\":[\"5878:static/chunks/9da6db1e-c7f8db75983a0531.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:static/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"7967:static/chunks/7967-ff831c2c55ce81b3.js\",\"3185:static/chunks/app/layout-861f2c7fab29b964.js\"],\"name\":\"SpeedInsights\",\"async\":false}\n18:I{\"id\":5245,\"chunks\":[\"5878:static/chunks/9da6db1e-c7f8db75983a0531.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:static/chunks/8475-73b121e7d1cde5b3.j"])</script><script>self.__next_f.push([1,"s\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"7967:static/chunks/7967-ff831c2c55ce81b3.js\",\"3185:static/chunks/app/layout-861f2c7fab29b964.js\"],\"name\":\"\",\"async\":false}\n19:I{\"id\":9034,\"chunks\":[\"5878:static/chunks/9da6db1e-c7f8db75983a0531.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:static/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"7967:static/chunks/7967-ff831c2c55ce81b3.js\",\"3185:static/chunks/app/layout-861f2c7fab29b964.js\"],\"name\":\"\",\"async\":false}\n1a:I{\""])</script><script>self.__next_f.push([1,"id\":9414,\"chunks\":[\"5878:static/chunks/9da6db1e-c7f8db75983a0531.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:static/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"7967:static/chunks/7967-ff831c2c55ce81b3.js\",\"3185:static/chunks/app/layout-861f2c7fab29b964.js\"],\"name\":\"\",\"async\":false}\n"])</script><script>self.__next_f.push([1,"4:[[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/7bdcace9eef5fb2c.css\",\"precedence\":\"next\"}]],[\"$\",\"$L7\",null,{\"buildId\":\"49DXczy-J-eIjveshnxJ7\",\"assetPrefix\":\"\",\"initialCanonicalUrl\":\"/blog/google-scale-authorization\",\"initialTree\":[\"\",{\"children\":[\"(main)\",{\"children\":[\"blog\",{\"children\":[[\"slug\",\"google-scale-authorization\",\"d\"],{\"children\":[\"__PAGE__\",{}]}]}]}]},\"$undefined\",\"$undefined\",true],\"initialHead\":[false,\"$L8\"],\"globalErrorComponent\":\"$9\",\"children\":[null,[\"$\",\"html\",null,{\"lang\":\"en\",\"className\":\"__variable_403957 __variable_f8bf1a\",\"children\":[[\"$\",\"head\",null,{\"children\":[[\"$\",\"link\",null,{\"rel\":\"icon\",\"type\":\"image/svg+xml\",\"href\":\"/favicon.svg\"}],[\"$\",\"link\",null,{\"rel\":\"icon\",\"type\":\"image/x-icon\",\"href\":\"/favicon.ico\"}],[\"$\",\"link\",null,{\"rel\":\"icon\",\"type\":\"image/png\",\"sizes\":\"32x32\",\"href\":\"/favicon-32x32.png\"}],[\"$\",\"link\",null,{\"rel\":\"icon\",\"type\":\"image/png\",\"sizes\":\"16x16\",\"href\":\"/favicon-16x16.png\"}],[\"$\",\"link\",null,{\"rel\":\"mask-icon\",\"href\":\"/safari-pinned-tab.svg\",\"type\":\"image/svg+xml\"}],[\"$\",\"link\",null,{\"rel\":\"preload\",\"href\":\"https://use.typekit.net/pjz2dde.css\",\"as\":\"style\"}],[\"$\",\"link\",null,{\"rel\":\"stylesheet\",\"href\":\"https://use.typekit.net/pjz2dde.css\"}],[\"$\",\"link\",null,{\"rel\":\"alternate\",\"type\":\"application/atom+xml\",\"title\":\"Atom Feed for authzed.com\",\"href\":\"/feed/atom\"}],[\"$\",\"link\",null,{\"rel\":\"alternate\",\"type\":\"application/rss+xml\",\"title\":\"RSS Feed for authzed.com\",\"href\":\"/feed/rss\"}],[\"$\",\"script\",null,{\"defer\":true,\"data-domain\":\"authzed.com\",\"src\":\"/js/script.js\",\"data-api\":\"/api/event\"}]]}],[\"$\",\"body\",null,{\"className\":\"bg-dark text-white overflow-x-hidden\",\"children\":[[[\"$\",\"style\",null,{\"children\":\"\\n div[data-consent-manager-dialog] button[type=submit] {\\n color: #fff;\\n background-color: rgb(133, 74, 170);\\n background-image: none;\\n }\\n \"}],[\"$\",\"div\",null,{\"id\":\"consent-manager\",\"className\":\"fixed left-6 bottom-6 mr-6 z-[2147483647] shadow-lg [\u0026\u003ediv]:rounded-lg [\u0026\u003ediv]:p-4 [\u0026\u003ediv]:pr-10 [\u0026\u003ediv]:bg-suns-1000\"}]],[\"$\",\"$La\",null,{\"children\":[[\"$\",\"$b\",null,{\"fallback\":null,\"children\":[\"$\",\"$Lc\",null,{\"children\":[\"$\",\"$Ld\",null,{}]}]}],[\"$\",\"$Le\",null,{\"children\":[[\"$\",\"$Lf\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"template\":[\"$\",\"$L10\",null,{}],\"templateStyles\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"childProp\":{\"current\":[null,[\"$\",\"div\",null,{\"className\":\"bg-dark text-white\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex announcement-bar relative flex-wrap justify-between gap-y-2.5 py-3 px-5 z-50 text-[.8125rem] text-white bg-rocks-700/85 opacity-100 border-rocks-500 border-b-2\",\"children\":[[\"$\",\"div\",null,{\"className\":\"announcment grid grid-flow-col auto-cols-2 tablet:flex tablet:flex-wrap gap-2 justify-items-start items-center\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex flex-initial\",\"dangerouslySetInnerHTML\":{\"__html\":\"\u003cp\u003eOffice Hours \u003cstrong\u003eFeb 27 at 9 AM PT / 12 PM ET\u003c/strong\u003e: Solving the Dual-Write Problem - A Deep Dive\u003c/p\u003e\"}}],[\"$\",\"$L11\",null,{\"href\":\"https://www.youtube.com/watch?v=6lDkXrFjuhc\",\"children\":[\"$\",\"button\",null,{\"className\":\"inline-flex items-center justify-center disabled:pointer-events-none disabled:opacity-50 focus:nonedata-[state=open]:bg-slate-100 bg-gradient-to-b from-[#FF5C61] to-[#9C3774] rounded-full border border-[#AC5184] transition-transform duration-100 hover:scale-[1.03] h-5 content-center text-[.75rem] font-medium px-3 py-2.5\",\"label\":\"Remind me \",\"icon\":\"bell\",\"children\":[\"$undefined\",\"Remind me \",[\"$\",\"svg\",null,{\"aria-hidden\":\"true\",\"focusable\":\"false\",\"data-prefix\":\"fas\",\"data-icon\":\"bell\",\"className\":\"svg-inline--fa fa-bell bg-transparent h-[0.625rem] w-[0.625rem] ml-2\",\"role\":\"img\",\"xmlns\":\"http://www.w3.org/2000/svg\",\"viewBox\":\"0 0 448 512\",\"style\":{},\"children\":[\"$\",\"path\",null,{\"fill\":\"currentColor\",\"d\":\"M224 0c-17.7 0-32 14.3-32 32V51.2C119 66 64 130.6 64 208v18.8c0 47-17.3 92.4-48.5 127.6l-7.4 8.3c-8.4 9.4-10.4 22.9-5.3 34.4S19.4 416 32 416H416c12.6 0 24-7.4 29.2-18.9s3.1-25-5.3-34.4l-7.4-8.3C401.3 319.2 384 273.9 384 226.8V208c0-77.4-55-142-128-156.8V32c0-17.7-14.3-32-32-32zm45.3 493.3c12-12 18.7-28.3 18.7-45.3H224 160c0 17 6.7 33.3 18.7 45.3s28.3 18.7 45.3 18.7s33.3-6.7 45.3-18.7z\",\"style\":{}}]}]]}]}]]}],[\"$\",\"div\",null,{\"className\":\"hidden laptop:flex gh-container flex-wrap flex-initial self-end gap-y-2.5\",\"children\":[[\"$\",\"span\",null,{\"children\":[\"$\",\"a\",null,{\"href\":\"https://github.com/authzed/spicedb\",\"className\":\"text-current no-underline\",\"children\":\"SpiceDB, Open Source Google Zanzibar FGA \"}]}],[\"$\",\"$L11\",null,{\"href\":\"https://github.com/authzed/spicedb\",\"children\":[\"$\",\"$L12\",null,{\"size\":\"xs\",\"starCount\":5402}]}]]}]]}],[\"$\",\"$L13\",null,{}],[\"$\",\"div\",null,{\"children\":[\"$\",\"$Lf\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"(main)\",\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"template\":[\"$\",\"$L10\",null,{}],\"templateStyles\":\"$undefined\",\"notFound\":[[\"$\",\"title\",null,{\"children\":\"404: This page could not be found.\"}],[\"$\",\"div\",null,{\"style\":{\"fontFamily\":\"system-ui,\\\"Segoe UI\\\",Roboto,Helvetica,Arial,sans-serif,\\\"Apple Color Emoji\\\",\\\"Segoe UI Emoji\\\"\",\"height\":\"100vh\",\"textAlign\":\"center\",\"display\":\"flex\",\"flexDirection\":\"column\",\"alignItems\":\"center\",\"justifyContent\":\"center\"},\"children\":[\"$\",\"div\",null,{\"children\":[[\"$\",\"style\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"body{color:#000;background:#fff;margin:0}.next-error-h1{border-right:1px solid rgba(0,0,0,.3)}@media (prefers-color-scheme:dark){body{color:#fff;background:#000}.next-error-h1{border-right:1px solid rgba(255,255,255,.3)}}\"}}],[\"$\",\"h1\",null,{\"className\":\"next-error-h1\",\"style\":{\"display\":\"inline-block\",\"margin\":\"0 20px 0 0\",\"padding\":\"0 23px 0 0\",\"fontSize\":24,\"fontWeight\":500,\"verticalAlign\":\"top\",\"lineHeight\":\"49px\"},\"children\":\"404\"}],[\"$\",\"div\",null,{\"style\":{\"display\":\"inline-block\"},\"children\":[\"$\",\"h2\",null,{\"style\":{\"fontSize\":14,\"fontWeight\":400,\"lineHeight\":\"49px\",\"margin\":0},\"children\":\"This page could not be found.\"}]}]]}]}]],\"notFoundStyles\":[],\"childProp\":{\"current\":[\"$\",\"$Lf\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"(main)\",\"children\",\"blog\",\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"template\":[\"$\",\"$L10\",null,{}],\"templateStyles\":\"$undefined\",\"notFound\":\"$undefined\",\"notFoundStyles\":\"$undefined\",\"childProp\":{\"current\":[\"$\",\"$Lf\",null,{\"parallelRouterKey\":\"children\",\"segmentPath\":[\"children\",\"(main)\",\"children\",\"blog\",\"children\",[\"slug\",\"google-scale-authorization\",\"d\"],\"children\"],\"loading\":\"$undefined\",\"loadingStyles\":\"$undefined\",\"hasLoading\":false,\"error\":\"$undefined\",\"errorStyles\":\"$undefined\",\"template\":[\"$\",\"$L10\",null,{}],\"templateStyles\":\"$undefined\",\"notFound\":\"$undefined\",\"notFoundStyles\":\"$undefined\",\"childProp\":{\"current\":[\"$L14\",\"$L15\",null],\"segment\":\"__PAGE__\"},\"styles\":[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/da72cc09e5cef6cf.css\",\"precedence\":\"next\"}]]}],\"segment\":[\"slug\",\"google-scale-authorization\",\"d\"]},\"styles\":[]}],\"segment\":\"blog\"},\"styles\":[]}]}]]}],null],\"segment\":\"(main)\"},\"styles\":[[\"$\",\"link\",\"0\",{\"rel\":\"stylesheet\",\"href\":\"/_next/static/css/f54ebf99cea804b7.css\",\"precedence\":\"next\"}]]}],[\"$\",\"$b\",null,{\"fallback\":\"$undefined\",\"children\":[\"$\",\"$L16\",null,{\"properties\":\"$undefined\"}]}]]}]]}],[\"$\",\"$L17\",null,{\"sampleRate\":0.8}],[\"$\",\"$L18\",null,{}],[\"$\",\"$b\",null,{\"fallback\":null,\"children\":[\"$\",\"$Lc\",null,{\"children\":[\"$\",\"$L19\",null,{}]}]}],[\"$\",\"$L1a\",null,{}]]}]]}],null]}]]\n"])</script><script>self.__next_f.push([1,"1b:I{\"id\":4244,\"chunks\":[\"3676:static/chunks/870fdd6f-9c571350bc2478c3.js\",\"3958:static/chunks/69b09407-0b4f7c53994b856f.js\",\"4724:static/chunks/4724-69a117ad21aeeee7.js\",\"3718:static/chunks/3718-0050314636e810ad.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:static/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"1279:static/chunks/1279-23ffe5d7b074e3d8.js\",\"3959:static/chunks/app/(main)/blog/[slug]/page-0b21ba72e4a8ca90.js\"],\"name\":\"\",\"async\":false}\n1c:I{\"id\":2602,\"ch"])</script><script>self.__next_f.push([1,"unks\":[\"3676:static/chunks/870fdd6f-9c571350bc2478c3.js\",\"3958:static/chunks/69b09407-0b4f7c53994b856f.js\",\"4724:static/chunks/4724-69a117ad21aeeee7.js\",\"3718:static/chunks/3718-0050314636e810ad.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:static/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"1279:static/chunks/1279-23ffe5d7b074e3d8.js\",\"3959:static/chunks/app/(main)/blog/[slug]/page-0b21ba72e4a8ca90.js\"],\"name\":\"\",\"async\":false}\n1d:I{\"id\":6964,\"chunks\":[\"3676:stati"])</script><script>self.__next_f.push([1,"c/chunks/870fdd6f-9c571350bc2478c3.js\",\"3958:static/chunks/69b09407-0b4f7c53994b856f.js\",\"4724:static/chunks/4724-69a117ad21aeeee7.js\",\"3718:static/chunks/3718-0050314636e810ad.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:static/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"1279:static/chunks/1279-23ffe5d7b074e3d8.js\",\"3959:static/chunks/app/(main)/blog/[slug]/page-0b21ba72e4a8ca90.js\"],\"name\":\"Image\",\"async\":false}\n1f:I{\"id\":6900,\"chunks\":[\"3676:static/chunks/870f"])</script><script>self.__next_f.push([1,"dd6f-9c571350bc2478c3.js\",\"3958:static/chunks/69b09407-0b4f7c53994b856f.js\",\"4724:static/chunks/4724-69a117ad21aeeee7.js\",\"3718:static/chunks/3718-0050314636e810ad.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:static/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"1279:static/chunks/1279-23ffe5d7b074e3d8.js\",\"3959:static/chunks/app/(main)/blog/[slug]/page-0b21ba72e4a8ca90.js\"],\"name\":\"NewsletterSignUp\",\"async\":false}\n20:I{\"id\":2065,\"chunks\":[\"3676:static/chunks/870fdd"])</script><script>self.__next_f.push([1,"6f-9c571350bc2478c3.js\",\"3958:static/chunks/69b09407-0b4f7c53994b856f.js\",\"4724:static/chunks/4724-69a117ad21aeeee7.js\",\"3718:static/chunks/3718-0050314636e810ad.js\",\"9733:static/chunks/9733-e2429ec2121e38e9.js\",\"8475:static/chunks/8475-73b121e7d1cde5b3.js\",\"902:static/chunks/902-109f5995e466cf6c.js\",\"1279:static/chunks/1279-23ffe5d7b074e3d8.js\",\"3959:static/chunks/app/(main)/blog/[slug]/page-0b21ba72e4a8ca90.js\"],\"name\":\"CtaBoxWide\",\"async\":false}\n1e:T8968,"])</script><script>self.__next_f.push([1,"\u003cscript src=\"https://cdn.jsdelivr.net/npm/chart.js\"\u003e\u003c/script\u003e\n\u003ch2 id=\"user-content-background\"\u003eBackground\u003c/h2\u003e\n\u003cp\u003eWe've been working hard to make our managed \u003ca href=\"https://authzed.com/products/authzed-dedicated?utm_source=blog\u0026#x26;utm_medium=website\u0026#x26;utm_campaign=load_test\"\u003eAuthZed Dedicated\u003c/a\u003e offering capable of scaling as \u003ca href=\"https://zanzibar.tech/2SuZL2BPQU:0:14\" target=\"_blank\"\u003eGoogle does\u003c/a\u003e—particularly since it’s in our messaging to prospects and customers as a core reason to choose AuthZed. One of those prospects challenged us to demonstrate our bold claims with a series of load tests that would culminate with 1 million requests per second against 100 billion relationships stored.\u003c/p\u003e\n\u003cp\u003eIf you want to understand the performance of \u003ca href=\"https://authzed.com/products/authzed-dedicated?utm_source=blog\u0026#x26;utm_medium=website\u0026#x26;utm_campaign=load_test\"\u003eAuthZed Dedicated\u003c/a\u003e configured with \u003ca href=\"https://www.cockroachlabs.com/product/\" target=\"_blank\"\u003eCockroachDB\u003c/a\u003e for data storage or are interested in running your own scale tests, you will find our experience useful. I cover our setup and methodology below, along with the results. \u003cstrong\u003eSpoiler:\u003c/strong\u003e 👉 we got to 1 million requests per second, with 1% writes while maintaining 5.76ms P95 on CheckPermission. 🎉\u003c/p\u003e\n\u003cdiv class=\"hs-cta-embed hs-cta-simple-placeholder hs-cta-embed-163937461080\" style=\"max-width:100%; max-height:100%; width:621px;height:221.0416717529297px\"\u003e\n \u003ca href=\"https://cta-service-cms2.hubspot.com/web-interactives/public/v1/track/redirect?encryptedPayload=AVxigLJytiMX3uGW4y%2BAZfiIFjRoB%2BFi4%2By4yCQ7D7JDveWF5%2B%2Bc1CeZo66q4GI0eUzWi3weBc52yFCLRI7Hi8zM9Ul1BwJbsTnptG6ucO%2BvYRhZxxYooMy1K5MxEsD0u80SPujc49I0rNNXWF95hLBJvMyOPGN%2FSclVfes9tRfLlfbaHzTaHg4%3D\u0026#x26;webInteractiveContentId=163937461080\u0026#x26;portalId=43591865\" target=\"_blank\" rel=\"noopener\"\u003e\n \u003cimg alt=\"A small request We are trying to get SpiceDB to 5k stars, can you help us out by starring the repository? It helps increase awareness of SpiceDB, our open-source, Google Zanzibar-inspired permissions database. \" src=\"https://no-cache.hubspot.com/cta/default/43591865/interactive-163937461080.png\" style=\"height: 100%; width: 100%; object-fit: fill\"\u003e\n \u003c/a\u003e\n\u003c/div\u003e\n\u003ch2 id=\"user-content-goals\"\u003eGoals\u003c/h2\u003e\n\u003cp\u003eThe goals of the test were to validate that SpiceDB exhibits similar latency metrics across a total of 13 tests with varying throughput and relationships stored, here is the full list:\u003c/p\u003e\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\u003ctable\u003e\u003cthead\u003e\u003ctr\u003e\u003cth\u003eTest\u003c/th\u003e\u003cth\u003eRelationships Stored\u003c/th\u003e\u003cth\u003e* Target QPS\u003c/th\u003e\u003c/tr\u003e\u003c/thead\u003e\u003ctbody\u003e\u003ctr\u003e\u003ctd\u003eT1\u003c/td\u003e\u003ctd\u003e10,000,000\u003c/td\u003e\u003ctd\u003e1,000\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT2\u003c/td\u003e\u003ctd\u003e10,000,000\u003c/td\u003e\u003ctd\u003e10,000\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT3\u003c/td\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e1,000\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT4\u003c/td\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e10,000\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT5\u003c/td\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e100,000\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT6\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e1,000\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT7\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e10,000\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT8\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e100,000\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT9\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e1,000,000\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT10\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e1,000\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT11\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e10,000\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT12\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e100,000\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT13\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e1,000,000\u003c/td\u003e\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e\n\u003cp\u003e*\u003cem\u003eIncludes 1% WriteRelationship, remainder are CheckPermission\u003c/em\u003e\u003c/p\u003e\n\u003cp\u003eWe had a couple of secondary goals as well:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003eCollect various data points to understand and visualize the linear growth of computational resources with respect to traffic and dataset size.\u003c/li\u003e\n\u003cli\u003eIdentify how long it takes to upload data into a CockroachDB cluster. Many companies looking at a permissions database for authorization have existing data that needs migrating, and a crucial part of planning a rollout of a system like SpiceDB is understanding how long importing relationship data may take.\u003c/li\u003e\n\u003c/ul\u003e\n\u003ch2 id=\"user-content-test-infrastructure-software-and-environment\"\u003eTest Infrastructure, Software, and Environment\u003c/h2\u003e\n\u003cp\u003eFor our tests, we used our \u003ca href=\"https://authzed.com/products/authzed-dedicated?utm_source=blog\u0026#x26;utm_medium=website\u0026#x26;utm_campaign=load_test\"\u003eAuthZed Dedicated\u003c/a\u003e product to deploy a SpiceDB Permissions System onto AWS, with \u003ca href=\"https://www.cockroachlabs.com/product/\" target=\"_blank\"\u003eCockroachDB\u003c/a\u003e as the backing datastore. AuthZed Dedicated leverages \u003ca href=\"https://aws.amazon.com/eks/\" target=\"_blank\"\u003eEKS\u003c/a\u003e to distribute pods across the available virtual machines. AuthZed Dedicated also isolates workloads onto distinct compute pools: one for our control plane, which runs supporting services like Envoy, Prometheus, Alert Manager, Grafana, etc, and another for SpiceDB itself. The earlier tests had Envoy, which manages load distribution within a SpiceDB cluster, running within each SpiceDB node, but we moved Envoy early on in our tests to its own pool within the control plane.\u003c/p\u003e\n\u003cp\u003eIn AuthZed Dedicated, SpiceDB is deployed as a cluster of pod replicas with CPU and memory resource limits. We prevent SpiceDB Clusters from scheduling multiple pods (replicas) onto a single node with anti-affinity rules, and nodes are placed across availability zones to reduce the blast radius of a failure. Additionally, our Kubernetes clusters utilize CPU pinning—SpiceDB pods have dedicated CPU cores to maximize the time a core works within the same context.\u003c/p\u003e\n\u003cp\u003eWe ended our tests with SpiceDB version 1.21.0, which included all of the improvements we implemented along the way (see: \u003ca href=\"#spicedb-improvements-along-the-way\"\u003eSpiceDB Improvements Along the Way\u003c/a\u003e). For CockroachDB, we leveraged CockroachDB Dedicated, with version v22.2.11. Optimizing CockroachDB connection management is covered in \u003ca href=\"https://authzed.com/blog/maximizing-cockroachdb-performance\"\u003eMaximizing CockroachDB Performance: Our Journey to 1 Million QPS\u003c/a\u003e.\u003c/p\u003e\n\u003cp\u003eTo generate test data and test load, we wrote Go programs, which we detail below.\u003c/p\u003e\n\u003ch3 id=\"user-content-aws-infrastructure\"\u003eAWS Infrastructure\u003c/h3\u003e\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\u003ctable\u003e\u003cthead\u003e\u003ctr\u003e\u003cth\u003eTest\u003c/th\u003e\u003cth\u003eRelationships\u003c/th\u003e\u003cth\u003eQPS\u003c/th\u003e\u003cth\u003eControl Plane\u003c/th\u003e\u003cth\u003eQTY\u003c/th\u003e\u003cth\u003eSpiceDB\u003c/th\u003e\u003cth\u003eQTY\u003c/th\u003e\u003c/tr\u003e\u003c/thead\u003e\u003ctbody\u003e\u003ctr\u003e\u003ctd\u003eT1\u003c/td\u003e\u003ctd\u003e10,000,000\u003c/td\u003e\u003ctd\u003e1,000\u003c/td\u003e\u003ctd\u003ec6a.xlarge - 4 vCPU\u003c/td\u003e\u003ctd\u003e1\u003c/td\u003e\u003ctd\u003ec6a.xlarge - 4 vCPU\u003c/td\u003e\u003ctd\u003e3\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT2\u003c/td\u003e\u003ctd\u003e10,000,000\u003c/td\u003e\u003ctd\u003e10,000\u003c/td\u003e\u003ctd\u003ec6a.xlarge - 4 vCPU\u003c/td\u003e\u003ctd\u003e1\u003c/td\u003e\u003ctd\u003ec6a.2xlarge - 8 vCPU\u003c/td\u003e\u003ctd\u003e4\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT3\u003c/td\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e1,000\u003c/td\u003e\u003ctd\u003ec6a.xlarge - 4 vCPU\u003c/td\u003e\u003ctd\u003e1\u003c/td\u003e\u003ctd\u003ec6a.xlarge - 4 vCPU\u003c/td\u003e\u003ctd\u003e6\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT4\u003c/td\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e10,000\u003c/td\u003e\u003ctd\u003ec6a.xlarge - 4 vCPU\u003c/td\u003e\u003ctd\u003e1\u003c/td\u003e\u003ctd\u003ec6a.2xlarge - 8 vCPU\u003c/td\u003e\u003ctd\u003e4\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT5\u003c/td\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e100,000\u003c/td\u003e\u003ctd\u003ec6a.xlarge - 4 vCPU\u003c/td\u003e\u003ctd\u003e1\u003c/td\u003e\u003ctd\u003ec6a.4xlarge - 16 vCPU\u003c/td\u003e\u003ctd\u003e12\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT6\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e1,000\u003c/td\u003e\u003ctd\u003ec6a.xlarge - 4 vCPU\u003c/td\u003e\u003ctd\u003e1\u003c/td\u003e\u003ctd\u003ec6a.xlarge - 4 vCPU\u003c/td\u003e\u003ctd\u003e3\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT7\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e10,000\u003c/td\u003e\u003ctd\u003ec6a.xlarge - 4 vCPU\u003c/td\u003e\u003ctd\u003e2\u003c/td\u003e\u003ctd\u003ec6a.2xlarge - 8 vCPU\u003c/td\u003e\u003ctd\u003e3\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT8\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e100,000\u003c/td\u003e\u003ctd\u003ec6a.4xlarge - 16 vCPU\u003c/td\u003e\u003ctd\u003e5\u003c/td\u003e\u003ctd\u003ec6a.4xlarge - 16 vCPU\u003c/td\u003e\u003ctd\u003e7\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT9\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e1,000,000\u003c/td\u003e\u003ctd\u003ec6a.4xlarge - 16 vCPU\u003c/td\u003e\u003ctd\u003e36\u003c/td\u003e\u003ctd\u003ec6a.8xlarge - 32 vCPU\u003c/td\u003e\u003ctd\u003e25\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT10\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e1,000\u003c/td\u003e\u003ctd\u003ec6a.xlarge - 4 vCPU\u003c/td\u003e\u003ctd\u003e4\u003c/td\u003e\u003ctd\u003ec6a.xlarge - 4 vCPU\u003c/td\u003e\u003ctd\u003e3\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT11\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e10,000\u003c/td\u003e\u003ctd\u003ec6a.xlarge - 4 vCPU\u003c/td\u003e\u003ctd\u003e5\u003c/td\u003e\u003ctd\u003ec6a.2xlarge - 8 vCPU\u003c/td\u003e\u003ctd\u003e5\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT12\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e100,000\u003c/td\u003e\u003ctd\u003ec6a.2xlarge - 8 vCPU\u003c/td\u003e\u003ctd\u003e9\u003c/td\u003e\u003ctd\u003ec6a.4xlarge - 16 vCPU\u003c/td\u003e\u003ctd\u003e7\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT13\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e1,000,000\u003c/td\u003e\u003ctd\u003ec6a.4xlarge - 16 vCPU\u003c/td\u003e\u003ctd\u003e35\u003c/td\u003e\u003ctd\u003ec6a.8xlarge - 32 vCPU\u003c/td\u003e\u003ctd\u003e21\u003c/td\u003e\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e\n\u003ch3 id=\"user-content-cockroachdb-infrastructure\"\u003eCockroachDB Infrastructure\u003c/h3\u003e\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\u003ctable\u003e\u003cthead\u003e\u003ctr\u003e\u003cth\u003eTest\u003c/th\u003e\u003cth\u003eRelationships\u003c/th\u003e\u003cth\u003eQPS\u003c/th\u003e\u003cth\u003eCockroachDB\u003c/th\u003e\u003cth\u003eAttached Storage\u003c/th\u003e\u003cth\u003eQTY\u003c/th\u003e\u003c/tr\u003e\u003c/thead\u003e\u003ctbody\u003e\u003ctr\u003e\u003ctd\u003eT1\u003c/td\u003e\u003ctd\u003e10,000,000\u003c/td\u003e\u003ctd\u003e1,000\u003c/td\u003e\u003ctd\u003em6i.2xlarge - 8 vCPU\u003c/td\u003e\u003ctd\u003e75 GiB\u003c/td\u003e\u003ctd\u003e4\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT2\u003c/td\u003e\u003ctd\u003e10,000,000\u003c/td\u003e\u003ctd\u003e10,000\u003c/td\u003e\u003ctd\u003em6i.4xlarge - 16 vCPU\u003c/td\u003e\u003ctd\u003e150 GiB\u003c/td\u003e\u003ctd\u003e3\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT3\u003c/td\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e1,000\u003c/td\u003e\u003ctd\u003em6i.4xlarge - 16 vCPU\u003c/td\u003e\u003ctd\u003e150 GiB\u003c/td\u003e\u003ctd\u003e3\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT4\u003c/td\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e10,000\u003c/td\u003e\u003ctd\u003em6i.4xlarge - 16 vCPU\u003c/td\u003e\u003ctd\u003e150 GiB\u003c/td\u003e\u003ctd\u003e3\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT5\u003c/td\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e100,000\u003c/td\u003e\u003ctd\u003em6i.4xlarge - 16 vCPU\u003c/td\u003e\u003ctd\u003e150 GiB\u003c/td\u003e\u003ctd\u003e5\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT6\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e1,000\u003c/td\u003e\u003ctd\u003em6i.xlarge - 4 vCPU\u003c/td\u003e\u003ctd\u003e150 GiB\u003c/td\u003e\u003ctd\u003e3\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT7\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e10,000\u003c/td\u003e\u003ctd\u003em6i.2xlarge - 8 vCPU\u003c/td\u003e\u003ctd\u003e150 GiB\u003c/td\u003e\u003ctd\u003e3\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT8\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e100,000\u003c/td\u003e\u003ctd\u003em6i.4xlarge - 16 vCPU\u003c/td\u003e\u003ctd\u003e150 GiB\u003c/td\u003e\u003ctd\u003e6\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT9\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e1,000,000\u003c/td\u003e\u003ctd\u003em6i.8xlarge - 32 vCPU\u003c/td\u003e\u003ctd\u003e150 GiB\u003c/td\u003e\u003ctd\u003e12\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT10\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e1,000\u003c/td\u003e\u003ctd\u003em6i.2xlarge - 8 vCPU\u003c/td\u003e\u003ctd\u003e2,363 GiB\u003c/td\u003e\u003ctd\u003e6\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT11\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e10,000\u003c/td\u003e\u003ctd\u003em6i.2xlarge - 8 vCPU\u003c/td\u003e\u003ctd\u003e2,363 GiB\u003c/td\u003e\u003ctd\u003e6\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT12\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e100,000\u003c/td\u003e\u003ctd\u003em6i.4xlarge - 16 vCPU\u003c/td\u003e\u003ctd\u003e2,363 GiB\u003c/td\u003e\u003ctd\u003e6\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT13\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e1,000,000\u003c/td\u003e\u003ctd\u003em6i.8xlarge - 32 vCPU\u003c/td\u003e\u003ctd\u003e2,363 GiB\u003c/td\u003e\u003ctd\u003e6\u003c/td\u003e\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e\n\u003ch2 id=\"user-content-slo-targets\"\u003eSLO Targets\u003c/h2\u003e\n\u003cp\u003eIn addition to the target queries per second for each test, we defined success criteria. Each test was measured across a 10-minute period and needed to meet the following:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003e\u003ca href=\"https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.PermissionsService.CheckPermission\" target=\"_blank\"\u003eCheckPermission\u003c/a\u003e P95 of less than 25ms\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.PermissionsService.WriteRelationships\" target=\"_blank\"\u003eWriteRelationship\u003c/a\u003e P95 of less than 100ms\u003c/li\u003e\n\u003cli\u003eBoth needed to maintain a success rate greater than or equal to 99.9%\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp\u003eWe built a Grafana dashboard to capture all of the relevant metrics and manually ported them into a spreadsheet for each test. We also captured the value of various other relevant metrics to validate our assumptions on how SpiceDB behaves under load, e.g., we were able to visualize how the subproblem cache-hit rate exhibited asymptotic behavior with respect to the API QPS.\u003c/p\u003e\n\u003ch2 id=\"user-content-spicedb-schema-and-anticipated-load\"\u003eSpiceDB Schema and Anticipated Load\u003c/h2\u003e\n\u003cp\u003eA key part of SpiceDB is the SpiceDB Schema—it defines how entities relate to each other and the permissions those relationships drive. A SpiceDB Schema’s design drives the load on a SpiceDB cluster and the infrastructure capacity needed to maintain target SLOs. The complexity is added in the form of sub-problems that SpiceDB calculates; the \u003ca href=\"https://authzed.com/blog/check-it-out-2#breaking-into-subproblems\"\u003eCheck it Out-2\u003c/a\u003e blog does an excellent job of describing sub-problems in SpiceDB.\u003c/p\u003e\n\u003cp\u003eAnother consideration is your use of SpiceDB’s RPCs—our tests focused on CheckPermission and WriteRelationships from the \u003ca href=\"https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.PermissionsService\" target=\"_blank\"\u003ePermissionsService\u003c/a\u003e. If your workload requires listing resources, you will introduce the use of \u003ca href=\"https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.PermissionsService.LookupResources\" target=\"_blank\"\u003eLookupResources\u003c/a\u003e, which would impact utilization and require additional infrastructure.\u003c/p\u003e\n\u003cp\u003eAll tests were conducted against a customer-provided SpiceDB Schema, which modeled permissions for a social networking platform. Similar to how you use ORM (object-relational mapping) in an object programming language to map an object to a relational database, the concepts end-users see in a client application, e.g., a web app, need to be described in the SpiceDB Schema in a way that better serves the authorization model targeted. Here is the authorization behavior captured in the SpiceDB Schema:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003eUsers can share public/private content\u003c/li\u003e\n\u003cli\u003eUsers can follow each other\u003c/li\u003e\n\u003cli\u003eUsers can block other users\u003c/li\u003e\n\u003cli\u003eUsers can have conversations associated with a piece of content\u003c/li\u003e\n\u003cli\u003ePublic content can be seen by any user, private content can only be seen by followers\u003c/li\u003e\n\u003c/ul\u003e\n\u003ciframe src=\"https://play.authzed.com/i/O0zcnXwtKDQ-/schema\" width=\"100%\" height=\"500\" style=\"border:none;padding-bottom:20px;\"\u003e\u003c/iframe\u003e\n\u003cp\u003eAs you can see, the schema had a healthy dose of unions, intersections, and exclusions, three levels of nesting (user, content, and content interaction), and usage of wildcards to gate visibility. Your SpiceDB Schema is likely different and introduces its own complexity, which will have an impact on infrastructure requirements. See the \u003ca href=\"https://authzed.com/docs/reference/schema-lang#operations\" target=\"_blank\"\u003eSpiceDB Schema reference\u003c/a\u003e for a full list of operators.\u003c/p\u003e\n\u003ch2 id=\"user-content-generating-test-data\"\u003eGenerating Test Data\u003c/h2\u003e\n\u003cp\u003eWe were provided with the number of users, content published, and interactions on the content as a baseline from which we derived the total number of relationships expected for each relation. Here is the breakdown:\u003c/p\u003e\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\u003ctable\u003e\u003cthead\u003e\u003ctr\u003e\u003cth\u003eRelationship\u003c/th\u003e\u003cth\u003e1M\u003c/th\u003e\u003cth\u003e100M\u003c/th\u003e\u003cth\u003e1B\u003c/th\u003e\u003cth\u003e100B\u003c/th\u003e\u003c/tr\u003e\u003c/thead\u003e\u003ctbody\u003e\u003ctr\u003e\u003ctd\u003euser#visibility@user:*#...\u003c/td\u003e\u003ctd\u003e2,604\u003c/td\u003e\u003ctd\u003e260,436\u003c/td\u003e\u003ctd\u003e2,604,364\u003c/td\u003e\u003ctd\u003e260,436,395\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003euser#visibility@user#follower\u003c/td\u003e\u003ctd\u003e2,604\u003c/td\u003e\u003ctd\u003e260,436\u003c/td\u003e\u003ctd\u003e2,604,365\u003c/td\u003e\u003ctd\u003e260,436,495\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003euser#self@user#...\u003c/td\u003e\u003ctd\u003e5,209\u003c/td\u003e\u003ctd\u003e520,873\u003c/td\u003e\u003ctd\u003e5,208,729\u003c/td\u003e\u003ctd\u003e520,872,890\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003euser#follower@user#...\u003c/td\u003e\u003ctd\u003e52,083\u003c/td\u003e\u003ctd\u003e5,208,329\u003c/td\u003e\u003ctd\u003e52,083,292\u003c/td\u003e\u003ctd\u003e5,208,329,201\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003einteraction#creator@user#...\u003c/td\u003e\u003ctd\u003e104,167\u003c/td\u003e\u003ctd\u003e10,416,658\u003c/td\u003e\u003ctd\u003e104,166,583\u003c/td\u003e\u003ctd\u003e10,416,658,302\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003einteraction#parent@content#...\u003c/td\u003e\u003ctd\u003e104,167\u003c/td\u003e\u003ctd\u003e10,416,658\u003c/td\u003e\u003ctd\u003e104,166,583\u003c/td\u003e\u003ctd\u003e10,416,658,302\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003econtent#visibility@user:*#...\u003c/td\u003e\u003ctd\u003e180,989\u003c/td\u003e\u003ctd\u003e18,098,892\u003c/td\u003e\u003ctd\u003e180,988,918\u003c/td\u003e\u003ctd\u003e18,098,891,756\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003econtent#visibility@user#follower\u003c/td\u003e\u003ctd\u003e183,594\u003c/td\u003e\u003ctd\u003e18,359,412\u003c/td\u003e\u003ctd\u003e183,594,125\u003c/td\u003e\u003ctd\u003e18,359,412,451\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003econtent#owner@user#...\u003c/td\u003e\u003ctd\u003e364,583\u003c/td\u003e\u003ctd\u003e36,458,304\u003c/td\u003e\u003ctd\u003e364,583,042\u003c/td\u003e\u003ctd\u003e36,458,304,207\u003c/td\u003e\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e\n\u003cp\u003eHowever, these numbers by themselves don’t tell us how content and interactions on that content are distributed across the users, for instance:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003eHow many pieces of content does the average user publish?\u003c/li\u003e\n\u003cli\u003eHow many followers does the average user have?\u003c/li\u003e\n\u003cli\u003eWhat about the number of interactions by content published?\u003c/li\u003e\n\u003c/ul\u003e\n\u003ch3 id=\"user-content-using-pareto-distribution-to-shape-the-graph\"\u003eUsing Pareto Distribution to Shape the Graph\u003c/h3\u003e\n\u003cp\u003eIf you think of popular social networks like Instagram, YouTube, or Twitter, it's not unreasonable to expect that some users are more active than others; some even generate content hourly and have millions of followers. We decided to follow a Pareto distribution to model the activity on a social network (see: \u003ca href=\"https://www.nature.com/articles/srep01783\" target=\"_blank\"\u003eOrigins of power-law degree distribution in the heterogeneity of human activity in social networks\u003c/a\u003e). Meaning 20% of the network’s users generate 80% of the activity, comments, etc.\u003c/p\u003e\n\u003cp\u003eBeing a graph database, when testing SpiceDB it’s important to consider the shape of your data and its impact on querying and performance. For example, some users in the graph may have very wide relations (think thousands of relationships), which would require a lot of subproblem dispatching. Very nested group membership can also be expensive to compute (and that’s the reason Zanzibar includes a component named \u003ca href=\"https://zanzibar.tech/21tieegnDR:0:2Y\" target=\"_blank\"\u003eLeopard Index\u003c/a\u003e to handle this efficiently).\u003c/p\u003e\n\u003cp\u003eWe wrote a dataset generator in Go that took the number of users, pieces of content, and interactions as input, created relationships using a Pareto distribution, and outputted an \u003ca href=\"https://avro.apache.org/\" target=\"_blank\"\u003eApache Avro\u003c/a\u003e file with the relationships represented in SQL, all compressed using \u003ca href=\"https://en.wikipedia.org/wiki/Snappy_(compression)\" target=\"_blank\"\u003eSnappy\u003c/a\u003e. The file was stored in S3 and transferred into the target CockroachDB cluster. We ran the program to generate each dataset: 10M, 100M, 1B, and 100B.\u003c/p\u003e\n\u003cp\u003eThis process kick-started our work to support bulk migration natively in the SpiceDB API. From version 1.22, SpiceDB exposes \u003ca href=\"https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.ExperimentalService.BulkImportRelationships\" target=\"_blank\"\u003ebulk_import\u003c/a\u003e and \u003ca href=\"https://buf.build/authzed/api/docs/main:authzed.api.v1#authzed.api.v1.ExperimentalService.BulkExportRelationships\" target=\"_blank\"\u003ebulk_export\u003c/a\u003e RPCs that massively accelerate an initial import.\u003c/p\u003e\n\u003ch2 id=\"user-content-generating-load\"\u003eGenerating Load\u003c/h2\u003e\n\u003cp\u003eThe workload was generated by defining a script with our in-house \u003ccode\u003ethumper\u003c/code\u003e tool. Thumper is a domain-specific load-test tool written in Go that takes a YAML file as input. The YAML file describes the sequence of steps to execute at a specified weight to emulate the API usage patterns of your applications. We’re exploring \u003ca href=\"https://github.com/authzed/spicedb/issues/1431\" target=\"_blank\"\u003eopen sourcing Thumper\u003c/a\u003e—please upvote if you’d like to see this.\u003c/p\u003e\n\u003cp\u003eFor CheckPermission, we were provided with the following distribution:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003e60% of requests check if the user can see a piece of content\u003c/li\u003e\n\u003cli\u003e30% of requests check if the user can see another user's profile\u003c/li\u003e\n\u003cli\u003e10% of requests check if the user can see an interaction\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp\u003eWe also added 1% of write requests to the above distribution. 1% is more than the \u003ca href=\"https://zanzibar.tech/2IudGpP-JZ:0:1O\" target=\"_blank\"\u003e0.19% Google’s Zanzibar paper documented as peak writes\u003c/a\u003e, but a lot of their usage comes from \u003ca href=\"https://zanzibar.tech/25xSoW13un:0.V3atsH9If:0\" target=\"_blank\"\u003eRead\u003c/a\u003e requests, suggesting Zanzibar is doing more than just authorization at Google.\u003c/p\u003e\n\u003cp\u003eThe load-test script issued \u003ccode\u003eCheckPermission\u003c/code\u003e with \u003ccode\u003econsistency\u003c/code\u003e of \u003ccode\u003eminimize_latency\u003c/code\u003e, and the quantization window remained at the default 5 seconds, with a max staleness of 100% (another 5s seconds of potential staleness). If you want to know more about SpiceDB caching strategies, check out Jake's blog post \u003ca href=\"https://authzed.com/blog/hotspot-caching-in-google-zanzibar-and-spicedb\"\u003eHotspot Caching in Google Zanzibar and SpiceDB\u003c/a\u003e. Jake’s post was inspired by one of the few performance bottlenecks we fixed as we started cranking up the QPS knob to some serious numbers.\u003c/p\u003e\n\u003ch3 id=\"user-content-modeling-daus\"\u003eModeling DAUs\u003c/h3\u003e\n\u003cp\u003eThe numbers above by themselves are insufficient as well, e.g., which users, content, and interactions would actually make the requests? To model a real-world scenario we created synthetic Daily Active Users (DAUs) with a \u003cstrong\u003esampling factor\u003c/strong\u003e. The sampling factor instructed the load-test generator to loop over a uniformly distributed sample of the user pool. For example, if we had 1,000 users and a sampling factor of 1%, the tool creates a pool of users with IDs 0, 100, 200... until user 1000, and then randomly samples from this pool of users.\u003c/p\u003e\n\u003cp\u003eThe sampling factor is a key setting in a load test, as it has a direct impact on the volume of relationships SpiceDB needs to load from the datastore, and informs the compute required to withstand a specific workload. After all, Zanzibar has a \u003ca href=\"https://zanzibar.tech/23iStt3hOC:0:40\" target=\"_blank\"\u003estrong focus on minimizing database access\u003c/a\u003e as much as possible, and many of its core design choices are at the service of sparing every last bit of database capacity.\u003c/p\u003e\n\u003ch2 id=\"user-content-results\"\u003eResults\u003c/h2\u003e\n\u003cp\u003eWe succeeded at not only scaling to 1M QPS and 100B relationships but achieved consistent performance across all tests. 🎉 Along the way we identified and implemented performance improvements to SpiceDB (detailed in \u003ca href=\"#spicedb-improvements-along-the-way\"\u003eSpiceDB Improvements Along the Way\u003c/a\u003e. Somethings to keep in mind with our results:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003eT1 through T5 had Envoy running on each SpiceDB node, we moved Envoy into an isolated control plane node pool after T5.\u003c/li\u003e\n\u003cli\u003eT9 introduced crossfade revisions (\u003ca href=\"https://github.com/authzed/spicedb/pull/1285\" target=\"_blank\"\u003ePR #1285\u003c/a\u003e), which significantly improved performance.\u003c/li\u003e\n\u003cli\u003eTests T10 through T13 were conducted using version 1.21.0, which incorporated all of the performance improvements we uncovered in the tests before it.\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp\u003eEarlier tests were not revisited as we improved SpiceDB and AuthZed Dedicated since the updates were implemented to achieve the scale in our final test of 1M QPS and 100B relationships stored.\u003c/p\u003e\n\u003ch3 id=\"user-content-1000-qps\"\u003e1,000 QPS\u003c/h3\u003e\n\u003ch4 id=\"user-content-checkpermission-latency\"\u003eCheckPermission Latency\u003c/h4\u003e\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\u003ctable\u003e\u003cthead\u003e\u003ctr\u003e\u003cth\u003eTest\u003c/th\u003e\u003cth\u003eRelationships\u003c/th\u003e\u003cth\u003eCache Hit\u003c/th\u003e\u003cth\u003eP95 Check Latency\u003c/th\u003e\u003cth\u003eP50 Check Latency\u003c/th\u003e\u003c/tr\u003e\u003c/thead\u003e\u003ctbody\u003e\u003ctr\u003e\u003ctd\u003eT1\u003c/td\u003e\u003ctd\u003e10,000,000\u003c/td\u003e\u003ctd\u003e33.20%\u003c/td\u003e\u003ctd\u003e19.7\u003c/td\u003e\u003ctd\u003e5.39\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT3\u003c/td\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e5.00%\u003c/td\u003e\u003ctd\u003e23.2\u003c/td\u003e\u003ctd\u003e7.03\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT6\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e21.60%\u003c/td\u003e\u003ctd\u003e18\u003c/td\u003e\u003ctd\u003e37\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT10\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e0.00%\u003c/td\u003e\u003ctd\u003e17.1\u003c/td\u003e\u003ctd\u003e8.05\u003c/td\u003e\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e\n\u003ch4 id=\"user-content-writerelationship-latency\"\u003eWriteRelationship Latency\u003c/h4\u003e\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\u003ctable\u003e\u003cthead\u003e\u003ctr\u003e\u003cth\u003eTest\u003c/th\u003e\u003cth\u003eRelationships\u003c/th\u003e\u003cth\u003eCache Hit\u003c/th\u003e\u003cth\u003eP95 Write Latency\u003c/th\u003e\u003cth\u003eP50 Write Latency\u003c/th\u003e\u003c/tr\u003e\u003c/thead\u003e\u003ctbody\u003e\u003ctr\u003e\u003ctd\u003eT1\u003c/td\u003e\u003ctd\u003e10,000,000\u003c/td\u003e\u003ctd\u003e33.20%\u003c/td\u003e\u003ctd\u003e30\u003c/td\u003e\u003ctd\u003e14.50\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT3\u003c/td\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e5.00%\u003c/td\u003e\u003ctd\u003e20.9\u003c/td\u003e\u003ctd\u003e13.00\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT6\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e21.60%\u003c/td\u003e\u003ctd\u003e53\u003c/td\u003e\u003ctd\u003e16.80\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT10\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e0.00%\u003c/td\u003e\u003ctd\u003e17.9\u003c/td\u003e\u003ctd\u003e12.90\u003c/td\u003e\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e\n\u003ch3 id=\"user-content-10000-qps\"\u003e10,000 QPS\u003c/h3\u003e\n\u003ch4 id=\"user-content-checkpermission-latency-1\"\u003eCheckPermission Latency\u003c/h4\u003e\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\u003ctable\u003e\u003cthead\u003e\u003ctr\u003e\u003cth\u003eTest\u003c/th\u003e\u003cth\u003eRelationships\u003c/th\u003e\u003cth\u003eCache Hit\u003c/th\u003e\u003cth\u003eP95 Check Latency\u003c/th\u003e\u003cth\u003eP50 Check Latency\u003c/th\u003e\u003c/tr\u003e\u003c/thead\u003e\u003ctbody\u003e\u003ctr\u003e\u003ctd\u003eT2\u003c/td\u003e\u003ctd\u003e10,000,000\u003c/td\u003e\u003ctd\u003e72.40%\u003c/td\u003e\u003ctd\u003e16.3\u003c/td\u003e\u003ctd\u003e3.49\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT4\u003c/td\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e72.40%\u003c/td\u003e\u003ctd\u003e14.8\u003c/td\u003e\u003ctd\u003e3.43\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT7\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e73.70%\u003c/td\u003e\u003ctd\u003e8.67\u003c/td\u003e\u003ctd\u003e3.28\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT11\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e65.10%\u003c/td\u003e\u003ctd\u003e10\u003c/td\u003e\u003ctd\u003e3.44\u003c/td\u003e\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e\n\u003ch4 id=\"user-content-writerelationship-latency-1\"\u003eWriteRelationship Latency\u003c/h4\u003e\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\u003ctable\u003e\u003cthead\u003e\u003ctr\u003e\u003cth\u003eTest\u003c/th\u003e\u003cth\u003eRelationships\u003c/th\u003e\u003cth\u003eCache Hit\u003c/th\u003e\u003cth\u003eP95 Write Latency\u003c/th\u003e\u003cth\u003eP50 Write Latency\u003c/th\u003e\u003c/tr\u003e\u003c/thead\u003e\u003ctbody\u003e\u003ctr\u003e\u003ctd\u003eT2\u003c/td\u003e\u003ctd\u003e10,000,000\u003c/td\u003e\u003ctd\u003e72.40%\u003c/td\u003e\u003ctd\u003e22.8\u003c/td\u003e\u003ctd\u003e13.20\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT4\u003c/td\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e72.40%\u003c/td\u003e\u003ctd\u003e22.8\u003c/td\u003e\u003ctd\u003e12.10\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT7\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e73.70%\u003c/td\u003e\u003ctd\u003e32.9\u003c/td\u003e\u003ctd\u003e13.80\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT11\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e65.10%\u003c/td\u003e\u003ctd\u003e37\u003c/td\u003e\u003ctd\u003e14.30\u003c/td\u003e\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e\n\u003ch3 id=\"user-content-100000-qps\"\u003e100,000 QPS\u003c/h3\u003e\n\u003ch4 id=\"user-content-checkpermission-latency-2\"\u003eCheckPermission Latency\u003c/h4\u003e\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\u003ctable\u003e\u003cthead\u003e\u003ctr\u003e\u003cth\u003eTest\u003c/th\u003e\u003cth\u003eRelationships\u003c/th\u003e\u003cth\u003eCache Hit\u003c/th\u003e\u003cth\u003eP95 Check Latency\u003c/th\u003e\u003cth\u003eP50 Check Latency\u003c/th\u003e\u003c/tr\u003e\u003c/thead\u003e\u003ctbody\u003e\u003ctr\u003e\u003ctd\u003eT5\u003c/td\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e87.60%\u003c/td\u003e\u003ctd\u003e11\u003c/td\u003e\u003ctd\u003e3.21\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT8\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e89.30%\u003c/td\u003e\u003ctd\u003e5.93\u003c/td\u003e\u003ctd\u003e3.12\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT12\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e89.20%\u003c/td\u003e\u003ctd\u003e5.86\u003c/td\u003e\u003ctd\u003e3.08\u003c/td\u003e\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e\n\u003ch4 id=\"user-content-writerelationship-latency-2\"\u003eWriteRelationship Latency\u003c/h4\u003e\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\u003ctable\u003e\u003cthead\u003e\u003ctr\u003e\u003cth\u003eTest\u003c/th\u003e\u003cth\u003eRelationships\u003c/th\u003e\u003cth\u003eCache Hit\u003c/th\u003e\u003cth\u003eP95 Write Latency\u003c/th\u003e\u003cth\u003eP50 Write Latency\u003c/th\u003e\u003c/tr\u003e\u003c/thead\u003e\u003ctbody\u003e\u003ctr\u003e\u003ctd\u003eT5\u003c/td\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e87.60%\u003c/td\u003e\u003ctd\u003e56.6\u003c/td\u003e\u003ctd\u003e17.70\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT8\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e89.30%\u003c/td\u003e\u003ctd\u003e42\u003c/td\u003e\u003ctd\u003e15.60\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT12\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e89.20%\u003c/td\u003e\u003ctd\u003e49.3\u003c/td\u003e\u003ctd\u003e14.90\u003c/td\u003e\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e\n\u003ch3 id=\"user-content-1000000-qps\"\u003e1,000,000 QPS\u003c/h3\u003e\n\u003ch4 id=\"user-content-checkpermission-latency-3\"\u003eCheckPermission Latency\u003c/h4\u003e\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\u003ctable\u003e\u003cthead\u003e\u003ctr\u003e\u003cth\u003eTest\u003c/th\u003e\u003cth\u003eRelationships\u003c/th\u003e\u003cth\u003eCache Hit\u003c/th\u003e\u003cth\u003eP95 Check Latency\u003c/th\u003e\u003cth\u003eP50 Check Latency\u003c/th\u003e\u003c/tr\u003e\u003c/thead\u003e\u003ctbody\u003e\u003ctr\u003e\u003ctd\u003eT9\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e95.70%\u003c/td\u003e\u003ctd\u003e5.89\u003c/td\u003e\u003ctd\u003e3.1\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT13\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e95.90%\u003c/td\u003e\u003ctd\u003e5.76\u003c/td\u003e\u003ctd\u003e3.03\u003c/td\u003e\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e\n\u003ch4 id=\"user-content-writerelationship-latency-3\"\u003eWriteRelationship Latency\u003c/h4\u003e\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\u003ctable\u003e\u003cthead\u003e\u003ctr\u003e\u003cth\u003eTest\u003c/th\u003e\u003cth\u003eRelationships\u003c/th\u003e\u003cth\u003eCache Hit\u003c/th\u003e\u003cth\u003eP95 Write Latency\u003c/th\u003e\u003cth\u003eP50 Write Latency\u003c/th\u003e\u003c/tr\u003e\u003c/thead\u003e\u003ctbody\u003e\u003ctr\u003e\u003ctd\u003eT9\u003c/td\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e95.70%\u003c/td\u003e\u003ctd\u003e74.1\u003c/td\u003e\u003ctd\u003e16.10\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003eT13\u003c/td\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e95.90%\u003c/td\u003e\u003ctd\u003e48.3\u003c/td\u003e\u003ctd\u003e15.80\u003c/td\u003e\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e\n\u003cp\u003eWe also collected data on how long it takes to import a large amount of relationships into CockroachDB—which seems very reasonable. The largest import of 100B relationships took just over 5 days, but we think this can be lowered by making sure the dataset is split evenly across smaller files. The import was achieved using the AVRO file format and Snappy compression.\u003c/p\u003e\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\n\u003ctable\u003e\u003cthead\u003e\u003ctr\u003e\u003cth\u003eRelationships\u003c/th\u003e\u003cth\u003e*Total Bytes per Relationship\u003c/th\u003e\u003cth\u003e**Total Relationship Storage\u003c/th\u003e\u003cth\u003eImport Time\u003c/th\u003e\u003c/tr\u003e\u003c/thead\u003e\u003ctbody\u003e\u003ctr\u003e\u003ctd\u003e10,000,000\u003c/td\u003e\u003ctd\u003e15\u003c/td\u003e\u003ctd\u003e0.39 GiB\u003c/td\u003e\u003ctd\u003e45s\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e100,000,000\u003c/td\u003e\u003ctd\u003e29\u003c/td\u003e\u003ctd\u003e7.85 GiB\u003c/td\u003e\u003ctd\u003e6m 20s\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e1,000,000,000\u003c/td\u003e\u003ctd\u003e42\u003c/td\u003e\u003ctd\u003e116.48 GiB\u003c/td\u003e\u003ctd\u003e2h 40m\u003c/td\u003e\u003c/tr\u003e\u003ctr\u003e\u003ctd\u003e100,000,000,000\u003c/td\u003e\u003ctd\u003e29\u003c/td\u003e\u003ctd\u003e7,948.15 GIB\u003c/td\u003e\u003ctd\u003e121h 24m\u003c/td\u003e\u003c/tr\u003e\u003c/tbody\u003e\u003c/table\u003e\n\u003cp\u003e\u003cem\u003e* Includes 3x replication factor, *\u003c/em\u003e Doesn’t include other CockroachDB system related storage*\u003c/p\u003e\n\u003cp\u003eThe graph below shows P95 CheckPermission and WriteRelationship latency, along with the observed Sub-Problem Cache Hit ratio, observed in the 4 tests conducted against 100B relationships stored: 1K, 10K, 100K, and 1M QPS.\u003c/p\u003e\n\u003cdiv class=\"relative\" style=\"min-height: 320px\"\u003e\u003ccanvas id=\"user-content-myChart\"\u003e\u003c/canvas\u003e\u003c/div\u003e\n\u003cscript defer\u003e\nvar ctx = document.getElementById('user-content-myChart').getContext('2d');\nvar myChart = new Chart(ctx, {\n type: 'line',\n data: {\n labels: ['T10', 'T11', 'T12', 'T13'],\n datasets: [{\n label: 'P95 CheckPermission Latency',\n data: [17.1,10,5.86,5.76],\n yAxisID: 'y',\n borderColor: '#E1769E',\n fill: false,\n pointBackgroundColor: '#E1769E'\n },{\n label: 'P95 WritesRelationship Latency',\n data: [17.9,37,49.3,48.3],\n yAxisID: 'y',\n borderColor: '#E9B674',\n fill: false,\n pointBackgroundColor: '#E9B674'\n },{\n label: 'Sub Problem Cache Hit Percentage',\n data: [0,.651,.892,.959],\n yAxisID: 'y1',\n borderColor: '#A7A2B3',\n fill: false,\n pointBackgroundColor: '#A7A2B3'\n }]\n },\n options: {\n maintainAspectRatio: false,\n scales: {\n y: {\n beginAtZero: true,\n title: {\n display: true,\n text: 'Latency (ms)'\n }\n },\n y1: {\n beginAtZero: true,\n position: 'right',\n title: {\n display: true,\n text: 'Sub Problem Cache Hit Rate'\n }\n },\n x: {\n title: {\n display: true,\n text: 'Test'\n }\n }\n },\n plugins: {\n title: {\n display: true,\n text: \"P95 Check and Write Latency Against 100B Relationships\",\n padding: {\n top: 10,\n bottom: 30\n }\n },\n legend: {\n position: 'bottom',\n labels: {\n color: '#07060a',\n boxWidth: 10,\n padding: 20,\n font: {\n size: 12,\n },\n usePointStyle: true\n }\n }\n }\n }\n});\n\u003c/script\u003e\n\u003ch3 id=\"user-content-spicedb-improvements-along-the-way\"\u003eSpiceDB Improvements Along the Way\u003c/h3\u003e\n\u003cp\u003eWhile working through the tests, we uncovered areas for improvement and implemented them as we proceeded. The list of improvements are:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003eCockroachDB connection balancing and pruning to reduce connection establishment overhead and better balance load across CockroachDB nodes (see \u003ca href=\"https://authzed.com/blog/maximizing-cockroachdb-performance\"\u003eMaximizing CockroachDB Performance: Our Journey to 1 Million QPS\u003c/a\u003e)\u003c/li\u003e\n\u003cli\u003eQuantization windows smearing to prevent connection-pool acquisition thundering herds at the end of a quantization window (see \u003ca href=\"https://authzed.com/blog/hotspot-caching-in-google-zanzibar-and-spicedb\"\u003eHotspot Caching in Google Zanzibar and SpiceDB\u003c/a\u003e)\u003c/li\u003e\n\u003cli\u003eFix to the hashring getting recomputed any time a sub-connection transitioned from ready to idle and vice-versa, which happened very frequently - \u003ca href=\"https://github.com/authzed/spicedb/pull/1310\" target=\"_blank\"\u003ePR #1310\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003eBetter balancing of dispatch operations in the cluster by increasing the replication factor, which meant all nodes received an equal amount of dispatch calls\u003c/li\u003e\n\u003cli\u003eUse static-CPU management in Kubernetes nodes\u003c/li\u003e\n\u003c/ul\u003e\n\u003ch3 id=\"user-content-additional-findings\"\u003eAdditional Findings\u003c/h3\u003e\n\u003cul\u003e\n\u003cli\u003eGenerating 100B relationships took 24 hours on a single thread.\u003c/li\u003e\n\u003cli\u003eGenerated AVRO files were 700GB.\u003c/li\u003e\n\u003cli\u003eOnce imported and indexed, our largest dataset of 100B relationships used a total of 8TB (this includes a replication factor of 3).\u003c/li\u003e\n\u003cli\u003e⭐ We reran T13 (not captured in the data above) with \u003ca href=\"https://aws.amazon.com/ec2/graviton/\" target=\"_blank\"\u003eAmazon’s ARM Graviton-based EC2 instances\u003c/a\u003e and observed 20% more throughput.\u003c/li\u003e\n\u003c/ul\u003e\n\u003ch2 id=\"user-content-lessons-learned\"\u003eLessons Learned\u003c/h2\u003e\n\u003cp\u003eA key thing to note is the interdependency between variables in the test and their impact on each other, e.g., an increase in the cache ratio decreases SpiceDB’s CPU % and CockroachDB’s QPS. Here is a partial causal loop diagram depicting the behavior we witnessed during the tests:\u003c/p\u003e\n\u003cp\u003e\u003cimg src=\"/images/causal-diagram.png\" alt=\"Causal loop diagram showing the interdependency of components in SpiceDB.\" title=\"SpiceDB Component Interdependency \"\u003e\u003c/p\u003e\n\u003cp\u003eWe also learned a lot about planning and executing large-scale load tests against CockroachDB. Here are some of our learnings:\u003c/p\u003e\n\u003cul\u003e\n\u003cli\u003eIf you are importing from S3 into CockroachDB, make sure the estimated import time does not exceed the expiration of an S3 pre-signed URL.\u003c/li\u003e\n\u003cli\u003eDisable incremental backups for the load test, as the first backup could take abnormally long and ruin the load-test session.\u003c/li\u003e\n\u003cli\u003eA CockroachDB Dedicated cluster can be scaled down to save costs while tests are not running. The duration is proportional to the number of nodes, as they have to be drained one by one and spun back up.\u003c/li\u003e\n\u003cli\u003eScaling in or out a cluster with this amount of data can take significant time and balloon costs, as data needs to be replicated and shards get rebalanced.\u003c/li\u003e\n\u003cli\u003eYou may need to get granular with your metrics to discover issues, e.g., we didn’t notice spikes in connection acquisition until we instructed Prometheus to scrape at 1-second intervals instead of the 30-second we were using.\u003c/li\u003e\n\u003cli\u003eDefault Spot-Instance limits can be easily hit at this scale, and with high machine count and market volatility, you may lose those savings if a test needs to be restarted.\u003c/li\u003e\n\u003c/ul\u003e\n\u003cp\u003eIf you have any questions about the test - don’t hesitate to reach out on our \u003ca href=\"https://authzed.com/discord\" target=\"_blank\"\u003eDiscord\u003c/a\u003e! I’m \u003cstrong\u003e@vroldanbet.\u003c/strong\u003e\u003c/p\u003e\n\u003ch2 id=\"user-content-additional-reading\"\u003eAdditional Reading\u003c/h2\u003e\n\u003cul\u003e\n\u003cli\u003e\u003ca href=\"https://authzed.com/blog/what-is-google-zanzibar\"\u003eUnderstanding Google Zanzibar: A Comprehensive Overview\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://authzed.com/zanzibar\"\u003eGoogle's Zanzibar White Paper, Annotated by AuthZed\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://authzed.com/blog/hotspot-caching-in-google-zanzibar-and-spicedb\"\u003eHotspot Caching in Google Zanzibar and SpiceDB\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://authzed.com/blog/maximizing-cockroachdb-performance\"\u003eMaximizing CockroachDB Performance: Our Journey to 1 Million QPS\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://authzed.com/blog/check-it-out-2#breaking-into-subproblems\"\u003eCheck it Out-2: Breaking Into Subproblems\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://authzed.com/products/authzed-dedicated?utm_source=blog\u0026#x26;utm_medium=website\u0026#x26;utm_campaign=load_test\"\u003eAuthZed Dedicated\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://authzed.com/blog/authz-primer\"\u003eA Primer on Modern Enterprise Authorization (AuthZ) Systems\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://authzed.com/blog/fine-grained-access-control\"\u003eFine-Grained Access Control: Can You Go Too Fine?\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://authzed.com/blog/exploring-rebac\"\u003eRelationship Based Access Control (ReBAC): Using Graphs to Power your Authorization System\u003c/a\u003e\u003c/li\u003e\n\u003cli\u003e\u003ca href=\"https://authzed.com/blog/pitfalls-of-jwt-authorization\"\u003ePitfalls of JWT Authorization\u003c/a\u003e\u003c/li\u003e\n\u003c/ul\u003e"])</script><script>self.__next_f.push([1,"15:[[\"$\",\"$L1b\",null,{\"src\":\"https://cdn.jsdelivr.net/npm/lite-youtube-embed@0.2.0/src/lite-yt-embed.min.js\",\"strategy\":\"lazyOnload\"}],[\"$\",\"$L1c\",null,{}],[\"$\",\"main\",null,{\"className\":\"w-full bg-light relative z-50 text-rocks-1000\",\"children\":[[\"$\",\"div\",null,{\"className\":\"relative flex place-content-center w-full overflow-hidden bg-cover bg-center\",\"children\":[[\"$\",\"div\",null,{\"className\":\"z-10 bg-slate-900/70 h-full w-full flex place-content-center\",\"children\":[\"$\",\"div\",null,{\"className\":\"relative z-10 py-8 laptop:py-16 flex flex-col gap-6 w-[90%] laptop:max-w-[1080px]\",\"children\":[[\"$\",\"div\",null,{\"className\":\"hidden laptop:flex items-center flex-wrap gap-1 tablet:gap-2\",\"children\":[[\"$\",\"$L11\",\"engineering\",{\"href\":\"/blog/tags/engineering\",\"className\":\"flex no-underline mt-0\",\"children\":[\"$\",\"span\",null,{\"className\":\"tag tag-lg hover:text-light\",\"children\":\"Engineering\"}]}],[\"$\",\"$L11\",\"product\",{\"href\":\"/blog/tags/product\",\"className\":\"flex no-underline mt-0\",\"children\":[\"$\",\"span\",null,{\"className\":\"tag tag-lg hover:text-light\",\"children\":\"Product\"}]}]]}],[\"$\",\"div\",null,{\"children\":[\"$\",\"h1\",null,{\"className\":\"mb-0 text-3xl tablet:text-5xl text-white\",\"children\":\"Google-Scale Authorization: Getting to 1 Million QPS on AuthZed Dedicated with CockroachDB\"}]}],[\"$\",\"div\",null,{\"className\":\"flex items-center flex-wrap gap-6 tablet:gap-0\",\"children\":[\"$\",\"div\",null,{\"className\":\"flex items-center gap-3\",\"children\":[[\"$\",\"div\",null,{\"className\":\"flex\",\"children\":[[\"$\",\"div\",\"/assets/team/victor-roldan-betancort.jpg\",{\"className\":\"relative inline-block h-[52px] w-[52px]\",\"style\":{\"zIndex\":10},\"children\":[\"$\",\"$L1d\",null,{\"src\":\"/assets/team/victor-roldan-betancort.jpg\",\"className\":\"object-fill rounded-full border-2 border-white\",\"alt\":\"/assets/team/victor-roldan-betancort.jpg\",\"fill\":true,\"sizes\":\"100vw\"}]}]]}],[\"$\",\"div\",null,{\"className\":\"flex flex-col gap-1\",\"children\":[[\"$\",\"div\",null,{\"className\":\"text-light\",\"children\":[\"$\",\"$L11\",null,{\"className\":\"font-medium hover:text-suns-800 transition-colors duration-150 ease-in-out\",\"href\":\"https://www.linkedin.com/in/vroldanbet/\",\"children\":\"Victor Roldan Betancort\"}]}],[\"$\",\"div\",null,{\"className\":\"flex text-light gap-2 font-regular text-sm\",\"children\":[[\"$\",\"span\",null,{\"children\":\"Updated September 19, 2024\"}],[\"$\",\"span\",null,{\"children\":\"|\"}],[\"$\",\"span\",null,{\"children\":\"21 min read\"}]]}]]}]]}]}]]}]}],[\"$\",\"$L1d\",null,{\"src\":\"/images/blogs/nasa-q1p7bh3shj8-unsplash.jpg\",\"className\":\"z-0 absolute\",\"fill\":true,\"sizes\":\"100vw\",\"style\":{\"objectFit\":\"cover\"},\"alt\":\"\",\"priority\":true,\"placeholder\":\"blur\",\"blurDataURL\":\"data:image/gif;base64,R0lGODlhAQABAPAAAIVKqv///yH5BAAAAAAALAAAAAABAAEAAAICRAEAOw==\"}]]}],[\"$\",\"div\",null,{\"className\":\"w-[90%] pt-8 content-default text-rocks-500 flex flex-col laptop:max-w-[1080px] laptop:pt-16 laptop:flex-row\",\"children\":[[\"$\",\"div\",null,{\"className\":\"w-full mb-24 prose tablet:prose-tablet max-w-none leading-[2rem] laptop:w-2/3 laptop:pr-20\",\"children\":[[\"$\",\"article\",null,{\"dangerouslySetInnerHTML\":{\"__html\":\"$1e\"}}],[\"$\",\"div\",null,{\"className\":\"mb-6 p-4 bg-rocks-100/70 rounded text-sm italic leading-relaxed\",\"children\":[\"Originally published \",\"July 12, 2023\",\"\"]}]]}],[\"$\",\"aside\",null,{\"className\":\"w-1/3 px-6 mb-6 text-rocks-1000 border-l sticky top-[80px] self-start ml-[1.25rem] max-h-[100-vh] overflow-y-auto\",\"children\":[\"$\",\"div\",null,{\"className\":\"hidden laptop:block\",\"children\":[[[\"$\",\"div\",null,{\"className\":\"text-body-sm font-bold mb-4\",\"children\":\"Table of Contents\"}],[\"$\",\"div\",null,{\"className\":\"flex flex-col gap-2 max-w-[300px]\",\"children\":[[\"$\",\"div\",\"Background\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#background\",\"children\":\"Background\"}]]}]}],[\"$\",\"div\",\"Goals\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#goals\",\"children\":\"Goals\"}]]}]}],[\"$\",\"div\",\"Test Infrastructure, Software, and Environment\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#test-infrastructure-software-and-environment\",\"children\":\"Test Infrastructure, Software, and Environment\"}]]}]}],[\"$\",\"div\",\"AWS Infrastructure\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#aws-infrastructure\",\"children\":\"AWS Infrastructure\"}]]}]}],[\"$\",\"div\",\"CockroachDB Infrastructure\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#cockroachdb-infrastructure\",\"children\":\"CockroachDB Infrastructure\"}]]}]}],[\"$\",\"div\",\"SLO Targets\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#slo-targets\",\"children\":\"SLO Targets\"}]]}]}],[\"$\",\"div\",\"SpiceDB Schema and Anticipated Load\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#spicedb-schema-and-anticipated-load\",\"children\":\"SpiceDB Schema and Anticipated Load\"}]]}]}],[\"$\",\"div\",\"Generating Test Data\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#generating-test-data\",\"children\":\"Generating Test Data\"}]]}]}],[\"$\",\"div\",\"Using Pareto Distribution to Shape the Graph\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#using-pareto-distribution-to-shape-the-graph\",\"children\":\"Using Pareto Distribution to Shape the Graph\"}]]}]}],[\"$\",\"div\",\"Generating Load\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#generating-load\",\"children\":\"Generating Load\"}]]}]}],[\"$\",\"div\",\"Modeling DAUs\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#modeling-daus\",\"children\":\"Modeling DAUs\"}]]}]}],[\"$\",\"div\",\"Results\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#results\",\"children\":\"Results\"}]]}]}],[\"$\",\"div\",\"1,000 QPS\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#1000-qps\",\"children\":\"1,000 QPS\"}]]}]}],[\"$\",\"div\",\"CheckPermission Latency\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#checkpermission-latency\",\"children\":\"CheckPermission Latency\"}]]}]}],[\"$\",\"div\",\"WriteRelationship Latency\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#writerelationship-latency\",\"children\":\"WriteRelationship Latency\"}]]}]}],[\"$\",\"div\",\"10,000 QPS\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#10000-qps\",\"children\":\"10,000 QPS\"}]]}]}],[\"$\",\"div\",\"CheckPermission Latency\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#checkpermission-latency-1\",\"children\":\"CheckPermission Latency\"}]]}]}],[\"$\",\"div\",\"WriteRelationship Latency\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#writerelationship-latency-1\",\"children\":\"WriteRelationship Latency\"}]]}]}],[\"$\",\"div\",\"100,000 QPS\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#100000-qps\",\"children\":\"100,000 QPS\"}]]}]}],[\"$\",\"div\",\"CheckPermission Latency\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#checkpermission-latency-2\",\"children\":\"CheckPermission Latency\"}]]}]}],[\"$\",\"div\",\"WriteRelationship Latency\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#writerelationship-latency-2\",\"children\":\"WriteRelationship Latency\"}]]}]}],[\"$\",\"div\",\"1,000,000 QPS\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#1000000-qps\",\"children\":\"1,000,000 QPS\"}]]}]}],[\"$\",\"div\",\"CheckPermission Latency\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#checkpermission-latency-3\",\"children\":\"CheckPermission Latency\"}]]}]}],[\"$\",\"div\",\"WriteRelationship Latency\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[2.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#writerelationship-latency-3\",\"children\":\"WriteRelationship Latency\"}]]}]}],[\"$\",\"div\",\"SpiceDB Improvements Along the Way\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#spicedb-improvements-along-the-way\",\"children\":\"SpiceDB Improvements Along the Way\"}]]}]}],[\"$\",\"div\",\"Additional Findings\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[1.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#additional-findings\",\"children\":\"Additional Findings\"}]]}]}],[\"$\",\"div\",\"Lessons Learned\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#lessons-learned\",\"children\":\"Lessons Learned\"}]]}]}],[\"$\",\"div\",\"Additional Reading\",{\"children\":[\"$\",\"div\",null,{\"className\":\"pl-[0.5rem] text-xs text-rocks-1000 flex gap-2 content-start items-start leading-[1.3125rem] transition-color duration-100 hover:text-suns-800\",\"children\":[[\"$\",\"div\",null,{\"children\":\"•\"}],[\"$\",\"a\",null,{\"href\":\"#additional-reading\",\"children\":\"Additional Reading\"}]]}]}]]}]],[\"$\",\"div\",null,{\"className\":\"mt-8\",\"children\":[[\"$\",\"div\",null,{\"className\":\"text-md font-bold\",\"children\":\"Subscribe to Our Newsletter\"}],[\"$\",\"div\",null,{\"className\":\"text-sm\",\"children\":\"Stay updated on new releases, features, and announcements.\"}],[\"$\",\"div\",null,{\"className\":\"mt-2\",\"children\":[\"$\",\"$L1f\",null,{\"id\":\"blog-hs-form\"}]}]]}]]}]}]]}],[\"$\",\"div\",null,{\"className\":\"w-[90%] content-default laptop:max-w-[1080px]\",\"children\":[\"$\",\"$L20\",null,{\"title\":\"See AuthZed in action\",\"subtitle\":\"Build delightful, secure application experiences with AuthZed.\",\"buttonLabel\":\"Book demo\",\"href\":\"/call\",\"clickEventName\":\"CTA Clicked\",\"clickEventProperties\":{\"az_cta_id\":\"blog_post_primary\",\"az_component\":\"CtaBoxWide\",\"az_content\":\"Book demo\"}}]}],[\"$\",\"$L1b\",null,{\"type\":\"application/ld+json\",\"id\":\"structured-data\",\"strategy\":\"afterInteractive\",\"children\":\"{\\n \\\"@context\\\": \\\"https://schema.org\\\",\\n \\\"@type\\\": \\\"BlogPosting\\\",\\n \\\"headline\\\": \\\"Google-Scale Authorization: Getting to 1 Million QPS on SpiceDB Dedicated with CockroachDB\\\",\\n \\\"name\\\": \\\"Google-Scale Authorization: Getting to 1 Million QPS on SpiceDB Dedicated with CockroachDB\\\",\\n \\\"image\\\": [\\n \\\"https://authzed.com/images/blogs/nasa-q1p7bh3shj8-unsplash.jpg\\\"\\n ],\\n \\\"datePublished\\\": \\\"2023-07-12T12:05:53.554Z\\\",\\n \\\"dateModified\\\": \\\"2024-09-19T09:34:00.000Z\\\",\\n \\\"author\\\": [{\\\"@type\\\":\\\"Person\\\",\\\"name\\\":\\\"\\\\\\\"Victor Roldan Betancort\\\\\\\"\\\",\\\"url\\\":\\\"https://www.linkedin.com/in/vroldanbet/\\\"}],\\n \\\"publisher\\\":\\n {\\n \\\"name\\\": \\\"AuthZed Blog\\\",\\n \\\"url\\\": \\\"https://authzed.com/blog\\\"\\n }\\n}\"}]]}],[\"$\",\"div\",null,{\"className\":\"w-full content-section bg-light\",\"children\":[\"$\",\"section\",null,{\"className\":\"pt-7 pb-14 md:py-20 content-default\",\"children\":[[\"$\",\"div\",null,{\"className\":\"w-full pb-14 border-t border-rocks-100\"}],[\"$\",\"div\",null,{\"className\":\"self-center w-full\",\"children\":[\"$\",\"div\",null,{\"className\":\"nav-links footer footer-light\",\"children\":[[\"$\",\"div\",\"group-0\",{\"className\":\"group\",\"children\":[[\"$\",\"div\",null,{\"className\":\"group-title\",\"children\":\"Products\"}],[\"$\",\"div\",null,{\"children\":[[[\"$\",\"div\",\"div-0-0\",{\"className\":\"group-header\",\"children\":[\"$\",\"span\",null,{\"children\":\"Managed Services\"}]}],[\"$\",\"div\",\"item-0-1\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/products/authzed-dedicated\",\"children\":\"AuthZed Dedicated\"}]}]],[[\"$\",\"div\",\"div-0-2\",{\"className\":\"group-header\",\"children\":[\"$\",\"span\",null,{\"children\":\"Self-Hosted\"}]}],[\"$\",\"div\",\"item-0-3\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/products/authzed-support\",\"children\":\"AuthZed Support\"}]}],[\"$\",\"div\",\"item-0-4\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/products/spicedb-enterprise\",\"children\":\"SpiceDB Enterprise\"}]}]],[[\"$\",\"div\",\"div-0-5\",{\"className\":\"group-header\",\"children\":[\"$\",\"span\",null,{\"children\":\"Open Source\"}]}],[\"$\",\"div\",\"item-0-6\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/spicedb\",\"children\":\"SpiceDB\"}]}],[\"$\",\"div\",\"item-0-7\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/products/spicedb-operator\",\"children\":\"SpiceDB Operator\"}]}],[\"$\",\"div\",\"item-0-8\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/products/spicedb-clients\",\"children\":\"SpiceDB Clients\"}]}]]]}]]}],[\"$\",\"div\",\"group-1\",{\"className\":\"group\",\"children\":[[\"$\",\"div\",null,{\"className\":\"group-title\",\"children\":\"Resources\"}],[\"$\",\"div\",null,{\"children\":[[[\"$\",\"div\",\"div-1-0\",{\"className\":\"group-header\",\"children\":[\"$\",\"span\",null,{\"children\":\"Learn\"}]}],[\"$\",\"div\",\"item-1-1\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"https://play.authzed.com\",\"children\":\"Playground\"}]}],[\"$\",\"div\",\"item-1-2\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/docs\",\"children\":\"Docs\"}]}],[\"$\",\"div\",\"item-1-3\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/tour\",\"children\":\"Tour\"}]}],[\"$\",\"div\",\"item-1-4\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/customers\",\"children\":\"Customer Stories\"}]}],[\"$\",\"div\",\"item-1-5\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/blog/what-is-google-zanzibar\",\"children\":\"Learn More About Google Zanzibar\"}]}]],[[\"$\",\"div\",\"div-1-6\",{\"className\":\"group-header\",\"children\":[\"$\",\"span\",null,{\"children\":\"Support\"}]}],[\"$\",\"div\",\"item-1-7\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"https://security.authzed.com\",\"children\":\"Security\"}]}],[\"$\",\"div\",\"item-1-8\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"https://status.authzed.com/\",\"children\":\"Status Page\"}]}],[\"$\",\"div\",\"item-1-9\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/create-ticket\",\"children\":\"Submit a Ticket\"}]}],[\"$\",\"div\",\"item-1-10\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/call\",\"children\":\"Schedule a Call\"}]}],[\"$\",\"div\",\"item-1-11\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"https://app.authzed.com\",\"children\":\"Serverless Login\"}]}]]]}]]}],[\"$\",\"div\",\"group-2\",{\"className\":\"group\",\"children\":[[\"$\",\"div\",null,{\"className\":\"group-title\",\"children\":\"Company\"}],[\"$\",\"div\",null,{\"children\":[[[\"$\",\"div\",\"div-2-0\",{\"className\":\"group-header\",\"children\":[\"$\",\"span\",null,{\"children\":\"Info\"}]}],[\"$\",\"div\",\"item-2-1\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/about\",\"children\":\"About Us\"}]}],[\"$\",\"div\",\"item-2-2\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/blog\",\"children\":\"Blog\"}]}],[\"$\",\"div\",\"item-2-3\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/contact-us\",\"children\":\"Contact\"}]}],[\"$\",\"div\",\"item-2-4\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"https://www.workatastartup.com/companies/authzed\",\"children\":\"Join the Team\"}]}],[\"$\",\"div\",\"div-2-5\",{\"className\":\"group-header\",\"children\":[\"$\",\"span\",null,{\"children\":\"Legal\"}]}],[\"$\",\"div\",\"item-2-6\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/privacy-policy\",\"children\":\"Privacy Policy\"}]}],[\"$\",\"div\",\"item-2-7\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"/terms-conditions\",\"children\":\"Terms \u0026 Conditions\"}]}]]]}]]}],[\"$\",\"div\",\"group-3\",{\"className\":\"group\",\"children\":[[\"$\",\"div\",null,{\"className\":\"group-title\",\"children\":\"Community\"}],[\"$\",\"div\",null,{\"children\":[[[\"$\",\"div\",\"item-3-0\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"https://authzed.com/discord\",\"children\":\"Discord\"}]}],[\"$\",\"div\",\"item-3-1\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"https://github.com/authzed\",\"children\":\"GitHub\"}]}],[\"$\",\"div\",\"item-3-2\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"https://twitter.com/authzed\",\"children\":\"Twitter\"}]}],[\"$\",\"div\",\"item-3-3\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"https://www.youtube.com/channel/UCFeSgZf0rPqQteiTQNGgTPg\",\"children\":\"YouTube\"}]}],[\"$\",\"div\",\"item-3-4\",{\"children\":[\"$\",\"$L11\",null,{\"href\":\"https://www.linkedin.com/company/authzed/\",\"children\":\"LinkedIn\"}]}]]]}]]}]]}]}],[\"$\",\"div\",null,{\"className\":\"my-12 px-6 flex flex-col tablet:flex-row gap-16 tablet:gap-0\",\"children\":[[\"$\",\"div\",null,{\"className\":\"justify-self-center\",\"children\":[\"$\",\"$L11\",null,{\"href\":\"/\",\"children\":[\"$\",\"$L1d\",null,{\"src\":\"/authzed-logo-multi-dark.svg\",\"alt\":\"AuthZed\",\"width\":142,\"height\":38.51}]}]}],[\"$\",\"div\",null,{\"className\":\"tablet:ml-auto flex gap-4\",\"children\":[[\"$\",\"$L1d\",null,{\"alt\":\"Cloud Native Computing Foundation\",\"src\":\"/assets/logo-cncf-color.svg\",\"width\":170,\"height\":48,\"className\":\"w-[170px] h-[48px] tablet:w-[170px] tablet:h-[48px]\"}],[\"$\",\"$L1d\",null,{\"alt\":\"SOC for Service Organizations\",\"src\":\"/assets/SOC_NonCPA.svg\",\"width\":170,\"height\":168,\"className\":\"w-[55px] h-[55px] tablet:w-[75px] tablet:h-[75px] self-end\"}]]}]]}]]}]}]]\n"])</script><script>self.__next_f.push([1,"8:[[\"$\",\"meta\",\"0\",{\"charSet\":\"utf-8\"}],[\"$\",\"title\",\"1\",{\"children\":\"Google-Scale Authorization: Getting to 1 Million QPS on SpiceDB Dedicated with CockroachDB\"}],[\"$\",\"meta\",\"2\",{\"name\":\"description\",\"content\":\"Achieve 1 million requests per second with 1% writes while maintaining 5.76ms P95 on CheckPermission. The SpiceDB Dedicated product was deployed on AWS with CockroachDB as the backing datastore.\"}],[\"$\",\"meta\",\"3\",{\"name\":\"viewport\",\"content\":\"width=device-width, initial-scale=1\"}],[\"$\",\"meta\",\"4\",{\"name\":\"robots\",\"content\":\"index, follow, nocache\"}],[\"$\",\"meta\",\"5\",{\"name\":\"googlebot\",\"content\":\"index, follow, noimageindex, max-video-preview:-1, max-image-preview:large, max-snippet:-1\"}],[\"$\",\"link\",\"6\",{\"rel\":\"canonical\",\"href\":\"https://authzed.com/blog/google-scale-authorization\"}],[\"$\",\"meta\",\"7\",{\"property\":\"og:title\",\"content\":\"Google-Scale Authorization: Getting to 1 Million QPS on SpiceDB Dedicated with CockroachDB\"}],[\"$\",\"meta\",\"8\",{\"property\":\"og:description\",\"content\":\"Achieve 1 million requests per second with 1% writes while maintaining 5.76ms P95 on CheckPermission. The SpiceDB Dedicated product was deployed on AWS with CockroachDB as the backing datastore.\"}],[\"$\",\"meta\",\"9\",{\"property\":\"og:image\",\"content\":\"https://authzed.com/images/blogs/nasa-q1p7bh3shj8-unsplash.jpg\"}],[\"$\",\"meta\",\"10\",{\"name\":\"twitter:card\",\"content\":\"summary_large_image\"}],[\"$\",\"meta\",\"11\",{\"name\":\"twitter:title\",\"content\":\"Google-Scale Authorization: Getting to 1 Million QPS on SpiceDB Dedicated with CockroachDB\"}],[\"$\",\"meta\",\"12\",{\"name\":\"twitter:description\",\"content\":\"Achieve 1 million requests per second with 1% writes while maintaining 5.76ms P95 on CheckPermission. The SpiceDB Dedicated product was deployed on AWS with CockroachDB as the backing datastore.\"}],[\"$\",\"meta\",\"13\",{\"name\":\"twitter:image\",\"content\":\"https://authzed.com/images/blogs/nasa-q1p7bh3shj8-unsplash.jpg\"}],[\"$\",\"meta\",\"14\",{\"name\":\"next-size-adjust\"}]]\n"])</script><script>self.__next_f.push([1,"14:null\n"])</script></body></html>