[{"_1":2,"_1695":-5,"_1696":-5},"loaderData",{"_3":4,"_1118":1119,"_1391":1392},"root",{"_5":6,"_11":12,"_13":14,"_275":276,"_786":787,"_1112":49,"_1113":1114,"_1115":-7,"_1116":49,"_1117":49},"requestInfo",{"_7":8,"_9":10},"origin","https://aiven.io","path","/developer/terraform-cookbook-kafka","locale","en","footerLinks",[15,102,174,231],{"_16":12,"_17":18,"_19":20,"_21":22,"_23":24,"_30":31,"_32":33,"_27":34,"_35":36,"_99":100,"_55":101},"__i18n_lang","_createdAt","2022-06-03T12:30:17Z","_id","8fa73b6a-b62a-467a-8f5e-fa8dbf37a913","_rev","g8eHHsLV7ci6mnlkLsHEKA","_system",{"_25":26},"base",{"_27":20,"_28":29},"id","rev","Ytv0u8p7Ex1SAq2Ta7wKvY","_type","footer","_updatedAt","2025-10-06T09:43:27Z","company","links",[37,57,64,71,78,85,92],{"_38":39,"_30":40,"_41":42,"_48":49,"_11":12,"_50":12,"_51":52,"_55":56},"_key","9aaaaf74c9b4","link","internalLink",{"_43":44,"_30":45,"_46":47},"_ref","c79a4329-7549-48d7-90d2-617da8070ffd","reference","_weak",true,"isPdf",false,"pageLocale","slug",{"_30":51,"_53":54},"current","/about","title","About",{"_38":58,"_30":40,"_41":59,"_48":49,"_11":12,"_50":12,"_51":61,"_55":63},"3e8def23f6ce",{"_43":60,"_30":45,"_46":47},"227fce02-f6f1-4e82-a166-820dac2c21f5",{"_30":51,"_53":62},"/open-source","Open source",{"_38":65,"_30":40,"_41":66,"_48":49,"_11":12,"_50":12,"_51":68,"_55":70},"ee891e1cb5dc",{"_43":67,"_30":45,"_46":47},"9ec37216-4c4e-4430-8abe-e15365bc9d55",{"_30":51,"_53":69},"/careers","Careers",{"_38":72,"_30":40,"_41":73,"_48":49,"_11":12,"_50":12,"_51":75,"_55":77},"1a6d9ef744a0ec6de128641c623d2194",{"_43":74,"_30":45,"_46":47},"5c8b6345-6c17-4a8c-8ff1-fa2e51c3764a",{"_30":51,"_53":76},"/sustainability","Sustainability",{"_38":79,"_30":40,"_41":80,"_48":49,"_11":12,"_50":12,"_51":82,"_55":84},"f2b146716597",{"_43":81,"_30":45,"_46":47},"0fb13a4b-e541-40c6-af0b-c03c3a71aee7",{"_30":51,"_53":83},"/aiven-modern-slavery-statement","Modern slavery statement",{"_38":86,"_30":40,"_41":87,"_48":49,"_11":12,"_50":12,"_51":89,"_55":91},"883c820817d1",{"_43":88,"_30":45,"_46":47},"668f580e-9996-4c28-8fbe-1c6151081584",{"_30":51,"_53":90},"/press","Press",{"_38":93,"_30":40,"_41":94,"_48":49,"_11":12,"_50":12,"_51":96,"_55":98},"802a009b976c",{"_43":95,"_30":45,"_46":47},"438575c3-65ad-44ab-97bc-e0ff42080205",{"_30":51,"_53":97},"/blog","Blog","order",1,"Company",{"_16":12,"_17":103,"_19":104,"_21":105,"_30":31,"_32":106,"_27":107,"_35":108,"_99":172,"_55":173},"2025-10-06T12:02:16Z","7fb8141e-61af-4c25-b9c7-b1a0d6eb71fc","fk6Hb4IGMjHjsrsSzGx2rU","2025-10-06T12:04:58Z","terms-policies",[109,116,123,130,137,144,151,158,165],{"_38":110,"_30":40,"_41":111,"_48":49,"_11":12,"_50":12,"_51":113,"_55":115},"4f77b507f2b9",{"_43":112,"_30":45},"174ba708-1e30-481f-ab87-32c8d8fac151",{"_30":51,"_53":114},"/terms","Terms",{"_38":117,"_30":40,"_41":118,"_48":49,"_11":12,"_50":12,"_51":120,"_55":122},"0cd8982af965",{"_43":119,"_30":45},"ef2b25b2-d677-46f7-9d13-288f493a0629",{"_30":51,"_53":121},"/sla","SLA",{"_38":124,"_30":40,"_41":125,"_48":49,"_11":12,"_50":12,"_51":127,"_55":129},"bda5bec2b176",{"_43":126,"_30":45},"ada8e193-beb3-4982-bfa1-27beb24bf969",{"_30":51,"_53":128},"/aup","AUP",{"_38":131,"_30":40,"_41":132,"_48":49,"_11":12,"_50":12,"_51":134,"_55":136},"a61838008eec",{"_43":133,"_30":45},"c6a6bde8-7a3d-42b2-aec7-b89f7feeb2bd",{"_30":51,"_53":135},"/dpa","Data processing",{"_38":138,"_30":40,"_41":139,"_48":49,"_11":12,"_50":12,"_51":141,"_55":143},"e36e5fa0b307",{"_43":140,"_30":45},"e0c3ba03-765a-4d8c-bebb-90619d512ee5",{"_30":51,"_53":142},"/privacy","Privacy",{"_38":145,"_30":40,"_41":146,"_48":49,"_11":12,"_50":12,"_51":148,"_55":150},"4eb6efe9c789",{"_43":147,"_30":45},"c68abf7c-f7d2-45a5-8fdd-5c1472be1a6f",{"_30":51,"_53":149},"/dsa-point-of-contact","DSA contact",{"_38":152,"_30":40,"_41":153,"_48":49,"_11":12,"_50":12,"_51":155,"_55":157},"b3987f298645",{"_43":154,"_30":45},"16319f9c-a9fd-40b6-a142-55abea8fb589",{"_30":51,"_53":156},"/cookies","Cookie policy",{"_38":159,"_30":40,"_41":160,"_48":49,"_11":12,"_50":12,"_51":162,"_55":164},"ab6dda937edd",{"_43":161,"_30":45},"939b07b2-481a-4f5d-87f7-ecf0d4b3a5aa",{"_30":51,"_53":163},"/website-terms","Website terms of use",{"_38":166,"_30":40,"_41":167,"_48":49,"_11":12,"_50":12,"_51":169,"_55":171},"ed4625d819e5",{"_43":168,"_30":45},"67241aaf-6872-4046-ba51-ca6a18c3d9aa",{"_30":51,"_53":170},"/privacy/california","Do not sell or share my personal information",2,"Legal",{"_16":12,"_17":175,"_19":176,"_21":22,"_23":177,"_30":31,"_32":33,"_27":180,"_35":181,"_99":229,"_55":230},"2022-08-25T12:26:12Z","60aba555-c31d-4ed8-9c66-bc67c162acbe",{"_25":178},{"_27":176,"_28":179},"by5cisBNJtYQpnxlVUUNnM","platform",[182,189,196,203,210,217,224],{"_38":183,"_30":40,"_41":184,"_48":49,"_11":12,"_50":12,"_51":186,"_55":188},"f0d4b887749b5783a7ad2747201ddf21",{"_43":185,"_30":45,"_46":47},"97422fba-5006-46b6-9389-f8d782b04bd5",{"_30":51,"_53":187},"/responsibility-matrix","Responsibility matrix",{"_38":190,"_30":40,"_41":191,"_48":49,"_11":12,"_50":12,"_51":193,"_55":195},"66d0b92f3bbf",{"_43":192,"_30":45,"_46":47},"4a5f09b0-1992-4697-8a26-33e4788fc5dd",{"_30":51,"_53":194},"/subprocessors","Subprocessors",{"_38":197,"_30":40,"_41":198,"_48":49,"_11":12,"_50":12,"_51":200,"_55":202},"89301c4a104d",{"_43":199,"_30":45,"_46":47},"02d777ab-2af1-4eac-b8e7-385a17dfadc5",{"_30":51,"_53":201},"/security-compliance","Security and compliance",{"_38":204,"_30":40,"_41":205,"_48":49,"_11":12,"_50":12,"_51":207,"_55":209},"f0bf8b8217418050d21e04750c73d77f",{"_43":206,"_30":45,"_46":47},"07b6e990-ab38-4eaf-b4e6-d8af0126535c",{"_30":51,"_53":208},"/resources","Resource library",{"_38":211,"_30":40,"_41":212,"_48":49,"_11":12,"_50":12,"_51":214,"_55":216},"dd9d1c7cac1a",{"_43":213,"_30":45,"_46":47},"17d882dc-fcff-4bab-b539-c811e713bfa7",{"_30":51,"_53":215},"/support-services","Support services",{"_38":218,"_30":40,"_41":219,"_48":49,"_11":12,"_50":12,"_51":221,"_55":223},"d2b56a2bae64",{"_43":220,"_30":45,"_46":47},"814b2c0a-94d5-4b79-a17f-1a1d5f54f10a",{"_30":51,"_53":222},"/changelog","Changelog",{"_38":225,"_30":40,"_226":227,"_48":49,"_11":-5,"_50":12,"_51":-5,"_55":228},"7793d73cf559","externalUrl","https://status.aiven.io/","Aiven status",3,"Platform",{"_16":12,"_17":232,"_19":233,"_21":234,"_23":235,"_30":31,"_32":238,"_27":239,"_35":240,"_99":273,"_55":274},"2022-08-25T12:28:47Z","71a0dfc1-2d45-4a36-9b93-4303bb56b716","g8eHHsLV7ci6mnlkLsL9ow",{"_25":236},{"_27":233,"_28":237},"by5cisBNJtYQpnxlVUUhhx","2025-10-06T09:47:09Z","contacts",[241,248,255,259,266],{"_38":242,"_30":40,"_41":243,"_48":49,"_11":12,"_50":12,"_51":245,"_55":247},"8f47cdfeb034",{"_43":244,"_30":45},"de53f46b-d056-463c-8d2a-4ee0480b9faf",{"_30":51,"_53":246},"/contact","Contact us",{"_38":249,"_30":40,"_41":250,"_48":49,"_11":12,"_50":12,"_51":252,"_55":254},"88288062ead0",{"_43":251,"_30":45},"8930abef-2819-4638-b11a-a3a2e33e18ff",{"_30":51,"_53":253},"/book-demo","Book a demo",{"_38":256,"_30":40,"_226":257,"_48":49,"_11":-5,"_50":12,"_51":-5,"_55":258},"ec531c7ff267","https://aiven.io/docs/platform/howto/support","Support",{"_38":260,"_30":40,"_41":261,"_48":49,"_11":12,"_50":12,"_51":263,"_55":265},"06006a438dfe",{"_43":262,"_30":45},"3b5806cc-b4b0-494c-b509-3c3e45527296",{"_30":51,"_53":264},"/invoice","Invoice address",{"_38":267,"_30":40,"_41":268,"_48":49,"_11":12,"_50":12,"_51":270,"_55":272},"a6f9ec925e08",{"_43":269,"_30":45},"5ee42446-6891-458b-bac8-fd25fa3fc6f2",{"_30":51,"_53":271},"/events","Events calendar",4,"Contact","sharedContent",{"_277":278,"_317":318,"_396":397,"_403":404,"_98":512,"_517":518,"_521":522,"_70":569,"_576":577,"_585":586,"_751":752},"Developer Center",{"_279":280,"_281":282,"_283":284,"_285":286,"_287":288,"_289":290,"_291":292,"_293":294,"_295":296,"_297":298,"_299":300,"_301":302,"_303":304,"_305":306,"_307":308,"_309":310,"_311":312,"_313":314,"_315":316},"labelUseCase","Use cases","labelTech","Technologies","searchPlaceholder","Search Developer Center","ctaViewAll","View all articles","titleTech","Explore by technology","titleUseCase","Explore by use case","mainHeroTitle","Aiven Developer Center","mainHeroSubtitle","Unleash your development potential with our comprehensive guide to the latest tools and resources in open source technology","noArticle","No articles yet","noSubCategory","No sub-category","featured","Featured","rssSubscribeText","Subscribe to RSS","tableOfContents","Table of contents","titleForum","Get in on the conversation","subtitleForum","Questions? Comments? Report a bug? Discuss in the forum!","ctaForum","Join the community","titleWorkshops","Explore our workshops","labelCollection","Collections","titleCollections","Latest collections","Service Diagram",{"_319":320,"_321":322,"_323":324,"_325":180,"_326":327,"_328":329,"_330":331,"_332":333,"_334":335,"_336":337,"_338":339,"_340":341,"_342":343,"_344":345,"_346":347,"_348":349,"_350":351,"_352":353,"_354":355,"_356":357,"_358":359,"_360":361,"_362":341,"_363":364,"_365":366,"_367":368,"_369":370,"_371":372,"_373":374,"_375":376,"_377":378,"_379":380,"_381":382,"_383":384,"_385":386,"_387":388,"_389":390,"_391":392,"_393":386,"_394":395},"logs","Logs","grafana","Analytics and monitoring","kafkaLine1","Real-time data streaming","kafkaLine2","postgresql","Fully managed SQL database","metrics","Metrics","opensearchLine1","In-memory NoSQL","opensearchLine2","database","datasource","Datasource","dashboards","Dashboards","metricsAivenLine1","Metrics from","metricsAivenLine2","Aiven services","mirrorMaker","Data replication service for cluster to cluster data replication","schemaRegistry","Schema Registry","flinkLine1","Streaming SQL engine for stateful","flinkLine2","stream processing","clickhouseLine1","Data warehouse for","clickhouseLine2","streaming data","kafkaConnectExport","Export data as event streams","kafkaConnectImport","Stream data as event streams","plugins","Plugins","logsAivenLine1","Logs management for","logsAivenLine2","flink","Real-time analytics and ETL","openSearchPluginLine1","15+ plugins &","openSearchPluginLine2","dashboard","replica","Read replica","integratedAivenServices","Integrated Aiven services","extensions","Extensions","m3Line1","Independently scalable","m3Line2","aggregation","klawLine1","Data governance tool for","klawLine2","Apache Kafka topic management","restApi","Rest API","kafkaConnectLine1","Integrate existing","kafkaConnectLine2","sink connectors","kafkaCoreLine1","Core event streaming","kafkaCoreLine2","framework","kafkaConnectSourceLine1","kafkaConnectSourceLine2","source connectors","Newsletter",{"_398":399,"_400":399,"_301":302,"_401":402},"newsletterTitle","Stay updated with Aiven","subscribeText","newsletterText","Subscribe for the latest news and insights on open source, Aiven offerings, and more.","Kafka Plan Finder",{"_405":406,"_407":408,"_409":410,"_411":412,"_413":414,"_415":416,"_417":418,"_419":420,"_421":422,"_423":424,"_425":426,"_427":428,"_429":430,"_431":431,"_432":433,"_434":435,"_436":437,"_438":439,"_440":441,"_442":435,"_443":444,"_445":446,"_447":448,"_449":450,"_451":452,"_453":454,"_455":456,"_457":458,"_459":460,"_461":462,"_463":464,"_465":466,"_467":468,"_469":470,"_471":472,"_473":474,"_475":476,"_477":478,"_479":480,"_481":482,"_483":484,"_485":486,"_487":488,"_489":490,"_491":492,"_493":494,"_495":496,"_497":498,"_499":500,"_501":498,"_502":503,"_504":505,"_506":507,"_508":509,"_510":511},"cloud","Cloud","region","Region","addons","Add-ons","kpfHeroTitle","Find an Aiven for Apache Kafka® plan that fits you","kpfHeroSubtitle","Enter your data requirements, and we'll find the Aiven for Apache Kafka® plan that best meets your needs.","planPricing","Plan pricing","planComparison","Plan comparison","kpfCalcTitle","Your data needs","kpfCalcTitleStep1","Data volume","messagesPerSecond","Messages/Sec","kpfMessagesPerSecondDesc","This is an estimate of the number of messages you are producing\nto your Kafka cluster every second. If you are not yet using Kafka,\nthis can be estimated by using your system monitoring tools to determine\nthe number of events occurring every second. For example, you may be\nreceiving 300 API calls per second as well as 10 batch jobs that run\nevery 10 minutes; this gives you an idea of the messages produced\nevery second when you take Kafka into use.","averageMessageSize","Average message size (KB)","kpfAverageMessageSizeDesc","Kafka Messages are typically small in size but the variation can be large.\nIn order to find out the average message size, you can count the number of \nmessages in a topic and calculate the size of the topic.\n\nCount messages in a topic\n`kafkacat -b :-C -t -e -q | wc -l`\n\nCalculate topic size \n`/bin/kafka-log-dirs.sh --describe --bootstrap-server`\n`: ∞ --topic-list `","or","incomingDataVolume","Incoming data volume","kpfIncomingDataVolumeDesc","Your expected incoming data volume is based on your number\nof messages per second times the average size of your messages.\nIf unsure how to estimate this value, we recommend discussing \nwith your engineering team.","mbPerSec","MB/sec","kpfCalcTitleStep2","Retention & Replication","retention","Retention","kpfRetentionDesc","unitOfTime","Unit of time","kpfUnitOfTimeDesc","This is the period of time you expect to keep messages within your\nKafka cluster. This can range from hours to months and will vary \ndepending on your use case.","replicationFactor","Replication factor","kpfReplicationFactorDesc","The replication factor is the number of copies you need of the same\nmessage within your cluster. Kafka is distributed by design and a \nhigher replication factor helps your data to be protected in case one \nof your brokers fails.\n\nIt is recommended that the minimum value is n-1 where n is the number \nof brokers in your cluster. For example, a cluster of 3 would have a \nminimum replication factor of 2.\n\nThis value changes as the cluster grows in size (it is rare that you\nwill need a replication factor of 29 in a 30 node cluster). We recommend \na minimum of 3 and you can configure this within your cluster.","kpfCalcTitleStep3","Compute & Storage estimation","kpfResultTitle","Based on your data needs, we suggest the following plans","kpfResultSubtitle","Suggested plans are intended as a starting point; you can always upgrade or downgrade later.","kpfResultDisclaimer","*The estimated monthly pricing does not include the price for additional required storage. You can easily calculate cost including additional storage within Aiven console. Note: The est. monthly price can vary depending on the chosen cloud providers.","plan","Plan","dedicatedVMs","Dedicated VMs","cpuPerVM","CPU per VM","ramPerVM","RAM per VM","totalStorage","Total storage","extraStorageTooltip","Within Aiven console, you can book additional storage by yourself without switching plans.","extraStorage","Extra storage","hourlyPrice","Hourly price","estMonthlyPrice","Est. monthly price","custom","Custom","customPlanDisclaimer","Annual commitments are eligible for a discount. Custom plans available, [contact sales](/contact?department=1306714).","customPlanContactDisclaimer","There are no standard plans for your specified data needs. Please [contact sales](/contact?department=1306714) so we can help you find a plan beyond the listed options.","ctaSignup","Get started for free","ctaRelevantPlans","See only relevant plans","ctaAllPlans","See all plans","required","Required","privateLinkTooltip","${price}/GB based on the consumed traffic","features","Features","errorIncomingDataVolumeMbps","Value has to be a number greater than or equal to 0.001","errorAverageMessageSizeKb","Value has to be an integer greater than or equal to 1","errorMessagesPerSecond","Value has to be a number greater than or equal to 1","errorRetention","errorReplicationFactor","Value has to be an integer between 1 and 10","ctaContact","Contact sales","tieredStorage","Tiered storage","tieredStorageTooltip","Within Aiven console, you can enable tiered storage","estMonthlyPriceTooltip","Does not include the price for additional required storage",{"_513":514,"_303":304,"_515":516},"relatedBlogs","Related blogs","relatedResources","Related resources","Event",{"_519":520},"thankyouHeadline","Thank you for your interest. We’ve received your information and will be in touch shortly.","Pricing",{"_523":524,"_525":526,"_527":528,"_529":530,"_531":247,"_532":533,"_405":406,"_407":408,"_534":535,"_483":484,"_536":537,"_538":539,"_540":540,"_541":541,"_542":543,"_544":460,"_461":462,"_463":464,"_465":466,"_545":546,"_467":468,"_473":474,"_475":476,"_409":410,"_547":548,"_549":550,"_551":552,"_491":492,"_553":554,"_555":556,"_469":470,"_504":505,"_485":486,"_487":488,"_479":480,"_481":482,"_557":558,"_559":560,"_508":509,"_561":562,"_563":564,"_565":566,"_510":511,"_567":568},"serviceTitle","[THE COMPLETE AIVEN PLATFORM](/platform)","kpfDisclaimer","Need help? Use our [Kafka plan finder](/kafka-plan-finder) tool to get a recommended Kafka® plan based on your unique data needs and requirements.","kpfLinkText","Kafka plan finder tool","contactDisclaimer","Data retention is only limited by storage. For information about annual discount or plan capacity beyond the listed plans, please [contact us](/contact)","contactLinkText","service","Service","planDisclaimer","Pay as you go. [Contact us](/contact) for plan capacity beyond the listed plans.","ctaPriceTier","View plans","startingFrom","starting from","month","hour","comparePlans","Compare plans","planTier","shards","Shards","stream","Streaming","analyze","Analyze","store","Databases","byoaTitle","Bring Your Own Cloud (BYOC) available for larger setups","byoaDesc","Run Aiven services in your own cloud account to achieve lower TCO. Our self-service [BYOC](https://aiven.io/byoc) automation simplifies deployment. [Contact us](/contact) for more information. BYOC is available during general availability.","staticIpTooltip","${price}/hour per IP. E.g., 3 VM requires 6 IPs","awsTgwTooltip","${price}/hour based on transit usage","pricingCalculatorText","Need help? Use our [Pricing Calculator](/pricing/calculator?product={productId}) to get a recommended plan based on your unique data needs and requirements.","serve","Apps","kafkaTieredStorageTooltip","Starting from ${price} per GB/hour","totalStorageTooltip","Min storage, expandable up to the max capacity with additional paid storage",{"_570":571,"_572":573,"_574":575},"EngineerBannerTitle","Experience Aiven Platform firsthand","EngineerBannerDescription","We invite you to explore and experience the [product](/platform) you'll shape. Enjoy a 30-days trial with $300 credits included, along with the availability of free plan. Explore the features and become our advocate in the developer community.","EngineerBannerCTATitle","Get started with Aiven Platform","Language",{"_578":578,"_579":579,"_580":580,"_581":581,"_582":582,"_583":583,"_584":584},"English","German","Spanish","French","Japanese","Korean","Portuguese","Common",{"_587":588,"_589":590,"_591":592,"_593":594,"_301":302,"_595":596,"_597":598,"_599":600,"_601":602,"_603":604,"_605":606,"_607":608,"_609":610,"_611":274,"_612":613,"_614":615,"_616":617,"_618":619,"_620":406,"_621":622,"_623":624,"_625":626,"_627":628,"_629":630,"_631":632,"_633":634,"_635":636,"_637":638,"_639":640,"_641":642,"_643":644,"_645":646,"_647":648,"_649":650,"_651":517,"_652":653,"_654":655,"_656":657,"_658":659,"_660":661,"_662":663,"_664":665,"_666":667,"_668":669,"_283":670,"_671":277,"_672":673,"_674":675,"_676":677,"_678":679,"_680":681,"_682":683,"_684":685,"_407":408,"_686":687,"_688":689,"_690":691,"_692":693,"_694":695,"_547":548,"_551":552,"_549":550,"_696":697,"_698":699,"_700":98,"_701":622,"_702":622,"_703":704,"_705":706,"_707":708,"_709":710,"_711":712,"_713":714,"_715":716,"_717":718,"_719":719,"_720":721,"_722":723,"_724":725,"_563":564,"_726":727,"_728":729,"_730":731,"_732":733,"_734":735,"_736":91,"_737":738,"_739":740,"_741":742,"_743":744,"_745":746,"_747":748,"_749":750},"footerText","Apache, Apache Kafka, Kafka, Apache Flink, Flink, Apache Cassandra, and Cassandra are either registered trademarks or trademarks of the Apache Software Foundation in the United States and/or other countries. ClickHouse is a registered trademark of ClickHouse, Inc. https://clickhouse.com. M3, M3 Aggregator, OpenSearch, AlloyDB Omni, PostgreSQL, MySQL, InfluxDB, Grafana, Dragonfly, Valkey, Thanos, Terraform, and Kubernetes are trademarks and property of their respective owners. *Redis is a registered trademark of Redis Ltd. and the Redis box logo is a mark of Redis Ltd. Any rights therein are reserved to Redis Ltd. Any use by Aiven is for referential purposes only and does not indicate any sponsorship, endorsement or affiliation between Redis and Aiven. All product and service names used in this website are for identification purposes only and do not imply endorsement.","footerSocialText","Let's connect","errorTitle","We're currently experiencing some technical difficulties","errorFourOhFourTitle","Page not found","errorSubtitle","is currently unavailable. Please try refreshing the page later.","readMore","Read more","errorFourOhFourSubtitle","Perhaps the [home page](/) can help? Or you can explore our [developer center](/developer) or [blog](/blog).","hostProviderTitle","Deploy on","rssFeed","RSS Feed","loadMore","Load more","pressRelease","Press release","pressRoom","Press room","contact","aivenInTheNews","Aiven in the news","yearFounded","Year founded","close","Close","readCaseStudy","Read case study","hostedOn","productsUsed","Product","getStartedToday","Get your first cluster online now","getDirection","Get direction","login","Log in","loadingPageTitle","Loading page","loadingPageMessage","This will only take a moment","openSourceServices","Services","tooling","Tooling","cloudProviders","Deploy","industry","Industry","filterBy","Filter by","clearAll","Clear all","show","Show","aivenFor","Aiven for","nowRunning","Now running","event","meetup","Meet up","webinar","Webinar","conference","Conference","livestream","Livestream","tweet","Tweet","post","Post","tools","Tools","changeTypes","Types","noResult","No result found","Search","devCenter","ebook","E-book","whitepaper","White paper","report","Report","tool","Tool","freePlanAvailable","Free plan available","ctaEventList","View all events","online","Online","integrations","Third-party integrations","compliance","Compliance","identity","Identity management","managed","Managed","securityNetwork","Security & Network","learnMore","Learn more","type","Type","blog","productUsed","product","products","Products","video","Live video","workshop","Workshop","watchWorkshop","Explore workshop","downloadWhitepaper","Download white paper","downloadEbook","Download e-book","watchWebinar","Watch webinar","watchVideo","Watch video","results","otherResourcesNote","Dive into our technical guides at [Developer Center](/developer) or get inspired by our [Customer Stories](/case-studies). \n\nStay updated with our monthly newsletters, bringing fresh content directly to your inbox.","share","Share","email","Email","downloadReport","Download report","podcast","Podcast","watchPodcast","Watch podcast","caseStudy","Case study","earlyAvailability","Early availability","press","footerAboutAivenText","Aiven is an AI-ready open source data platform that combines open-choice services to rapidly stream, store and serve data across major cloud providers — simply and securely.","skipNav","Skip to content","topic","Topic","errorTooManyRequestsTitle","Too many requests","errorTooManyRequestsSubtitle","Too many requests, please try again later.","generalAvailability","General availability","alsoExploreOurOtherProducts","Also explore our other products","Case Studies",{"_623":624,"_753":754,"_755":254,"_756":757,"_758":759,"_760":761,"_762":763,"_764":765,"_766":767,"_768":769,"_770":771,"_483":484,"_772":773,"_774":775,"_776":777,"_778":779,"_780":781,"_782":783,"_784":785},"getStartedTodayText","Aiven makes setting up cloud databases so simple anyone can do it. Our set-it-and-forget-it solutions take the pain out of cloud data infrastructure.","bookADemo","relatedCaseStudies","Related case studies","exploreMoreCasesLink","browse all Aiven case studies","exploreMoreCases","Explore more customer success stories relevant to this one or","downloadPDF","Download case study (PDF)","allCaseStudies","All case studies","caseStudies","case studies","readCaseStudyLink","Read the full case study","getAccess","Get access","navCustomer","Customer spotlight","navCaseStudies","Case studies","csHeroTitle","Explore Aiven's customer stories, insightful videos, and beyond","resources","Resources","ctaReadCaseStudy","Read the case study","ctaReadBlog","Read the blog post","aboutTheCompany","About the company","navigationData",{"_16":12,"_17":788,"_19":789,"_21":790,"_23":791,"_30":794,"_32":795,"_796":797,"_1100":1101,"_1104":1105,"_51":1108,"_55":1110,"_828":1111},"2022-08-25T12:18:32Z","4f7b3f3c-98b7-4feb-8429-063bcf5d3ddf","x4upSj2I2b2gSerpXokgt1",{"_25":792},{"_27":789,"_28":793},"CyPycqJUJHqqULKBCh0kCL","navigation","2025-10-06T12:10:50Z","items",[798,952,1013,1096],{"_38":799,"_30":800,"_801":802,"_951":-5,"_55":704},"bb686c004610","navSubmenu","navLinkGroups",[803,854],{"_38":804,"_30":805,"_806":807,"_796":808,"_55":548},"d6637c001255","navLinkGroup","group","column-1",[809,830,842],{"_38":810,"_30":811,"_812":813,"_820":-5,"_821":822,"_828":829},"d4a24bea476c","navItem","icon",{"_30":814,"_815":816,"_817":818},"mainImage","alt","Aiven for Apache Kafka® logo","asset",{"_43":819,"_30":45},"image-1fb66f7813f0e69e21200bb5a43f630052bea86d-64x64-svg","image","navLink",{"_30":40,"_41":823,"_48":49,"_11":12,"_50":12,"_51":825,"_55":827},{"_43":824,"_30":45,"_46":47},"5e18872b-5613-4fc8-aaaa-0a36263634f5",{"_30":51,"_53":826},"/kafka","Aiven for Apache Kafka®","variant","default",{"_38":831,"_30":811,"_812":832,"_820":-5,"_821":836,"_828":829},"b2857faa7074",{"_30":814,"_815":833,"_817":834},"Aiven for Apache Flink® powered by Ververica logo",{"_43":835,"_30":45},"image-f7611d165ce0e31959431ac828afd1f98fe3c9ac-64x64-svg",{"_30":40,"_41":837,"_48":49,"_11":12,"_50":12,"_51":839,"_55":841},{"_43":838,"_30":45,"_46":47},"53f983e1-0b44-4570-8e1d-2f495cea3f7c",{"_30":51,"_53":840},"/flink","Aiven for Apache Flink®",{"_38":843,"_30":811,"_812":844,"_820":-5,"_821":848,"_828":829},"e0017a55de6a",{"_30":814,"_815":845,"_817":846},"Diskless for Apache Kafka® (BYOC) logo",{"_43":847,"_30":45},"image-46c9b452ab52c67e5c89ecaec9ed38f0b6b919b6-48x48-svg",{"_30":40,"_41":849,"_48":49,"_11":12,"_50":12,"_51":851,"_55":853},{"_43":850,"_30":45,"_46":47},"ae3c5ac2-13f4-4a62-81ef-bdc99d997b1f",{"_30":51,"_53":852},"/diskless","Diskless for Apache Kafka® (BYOC)",{"_38":855,"_30":805,"_806":856,"_796":857,"_55":552},"bc1c66122dd5","column-2",[858,870,882,893,905,917,929,940],{"_38":859,"_30":811,"_812":860,"_820":-5,"_821":864,"_828":829},"a7fdf13581dd",{"_30":814,"_815":861,"_817":862},"Aiven for PostgreSQL® logo",{"_43":863,"_30":45},"image-beacbd5dc1f66121bea2e56549e67f93409e2255-24x24-svg",{"_30":40,"_41":865,"_48":49,"_11":12,"_50":12,"_51":867,"_55":869},{"_43":866,"_30":45,"_46":47},"a829737a-0ab2-4037-a7af-be5f9a4f82cf",{"_30":51,"_53":868},"/postgresql","Aiven for PostgreSQL®",{"_38":871,"_30":811,"_812":872,"_820":-5,"_821":876,"_828":829},"4c31028377fcdb142abe3eacf4811a12",{"_30":814,"_815":873,"_817":874},"Aiven for AlloyDB Omni logo",{"_43":875,"_30":45},"image-3a5a57b209d20e380805e41238b337c0db0f21db-64x64-svg",{"_30":40,"_41":877,"_48":49,"_11":12,"_50":12,"_51":879,"_55":881},{"_43":878,"_30":45,"_46":47},"591f7c48-ecc0-4cb2-b7e5-c2aac4fea3aa",{"_30":51,"_53":880},"/opensearch","Aiven for OpenSearch®",{"_38":883,"_30":811,"_812":884,"_820":-5,"_821":887,"_828":829},"3afe4da565d1",{"_30":814,"_815":873,"_817":885},{"_43":886,"_30":45},"image-2745e6d516255592fdb3b785351c1c576f1c9e4c-64x64-svg",{"_30":40,"_41":888,"_48":49,"_11":12,"_50":12,"_51":890,"_55":892},{"_43":889,"_30":45,"_46":47},"37a2d4d0-b76b-41c6-9f9d-973358ee862d",{"_30":51,"_53":891},"/clickhouse","Aiven for ClickHouse®",{"_38":894,"_30":811,"_812":895,"_820":-5,"_821":899,"_828":829},"97bd70d16a2c",{"_30":814,"_815":896,"_817":897},"Aiven for Valkey™ logo",{"_43":898,"_30":45},"image-dafedbd81ff59bd7cba0b94fb091d00e360595ae-48x48-svg",{"_30":40,"_41":900,"_48":49,"_11":12,"_50":12,"_51":902,"_55":904},{"_43":901,"_30":45,"_46":47},"468bc3bf-30d4-4b6f-b021-d9a1bee1a7eb",{"_30":51,"_53":903},"/valkey","Aiven for Valkey™",{"_38":906,"_30":811,"_812":907,"_820":-5,"_821":911,"_828":829},"5ad670a9a200",{"_30":814,"_815":908,"_817":909},"Aiven for MySQL logo",{"_43":910,"_30":45},"image-9b64b9b4a39a8dcec4f509bae0bff3ee70e2fde7-64x64-svg",{"_30":40,"_41":912,"_48":49,"_11":12,"_50":12,"_51":914,"_55":916},{"_43":913,"_30":45,"_46":47},"e7fd6ec7-3b26-4955-abc0-77fcbbe307e2",{"_30":51,"_53":915},"/mysql","Aiven for MySQL",{"_38":918,"_30":811,"_812":919,"_820":-5,"_821":923,"_828":829},"04b7ece07fd465693d9e157394dce03a",{"_30":814,"_815":920,"_817":921},"Aiven for Dragonfly logo",{"_43":922,"_30":45},"image-99ebc9a00c29367d9df5517ef4474d89b5cc5b25-56x56-svg",{"_30":40,"_41":924,"_48":49,"_11":12,"_50":12,"_51":926,"_55":928},{"_43":925,"_30":45,"_46":47},"068b5598-f4e4-416e-9357-f797ca9348f8",{"_30":51,"_53":927},"/dragonfly","Aiven for Dragonfly",{"_38":930,"_30":811,"_812":931,"_820":-5,"_821":934,"_828":829},"ec77532e2002696a44d311c5cfec8545",{"_30":814,"_815":920,"_817":932},{"_43":933,"_30":45},"image-6fc7951e23dd2794ea0236fd1d74cb2d77583553-48x48-svg",{"_30":40,"_41":935,"_48":49,"_11":12,"_50":12,"_51":937,"_55":939},{"_43":936,"_30":45,"_46":47},"af7d02c4-3523-4a21-a721-2cd5fb32a0f4",{"_30":51,"_53":938},"/metrics","Aiven for Metrics",{"_38":941,"_30":811,"_812":942,"_820":-5,"_821":945,"_828":829},"15cd891bcb501e4b865fb864cb3943a8",{"_30":814,"_815":920,"_817":943},{"_43":944,"_30":45},"image-41576dff5ba0a23caf1f8d25fd6da0be3eb582a1-64x64-svg",{"_30":40,"_41":946,"_48":49,"_11":12,"_50":12,"_51":948,"_55":950},{"_43":947,"_30":45,"_46":47},"42c18edd-bcf8-4b56-8150-12787a872b60",{"_30":51,"_53":949},"/grafana","Aiven for Grafana®","navSubmenuItems",{"_38":953,"_30":800,"_801":954,"_951":-5,"_55":1012},"1aa58a277828",[955,975],{"_38":956,"_30":805,"_806":807,"_796":957,"_55":974},"ff81da24a7e9",[958,966],{"_38":959,"_30":811,"_820":-5,"_821":960,"_828":829},"6ee29dd78d50",{"_30":40,"_41":961,"_48":49,"_11":12,"_50":12,"_51":963,"_55":965},{"_43":962,"_30":45,"_46":47},"c988ff38-30ef-409e-b3ea-c3478c3b664e",{"_30":51,"_53":964},"/cluster","Startup",{"_38":967,"_30":811,"_820":-5,"_821":968,"_828":829},"c66c61b2b3ec",{"_30":40,"_41":969,"_48":49,"_11":12,"_50":12,"_51":971,"_55":973},{"_43":970,"_30":45,"_46":47},"a86fa077-8308-41ae-a33c-498d64b1d4db",{"_30":51,"_53":972},"/partners","Partners","By stage",{"_38":976,"_30":805,"_806":856,"_796":977,"_55":1011},"541556945b60",[978,986,994,1002],{"_38":979,"_30":811,"_820":-5,"_821":980,"_828":829},"eeb765096269",{"_30":40,"_41":981,"_48":49,"_11":12,"_50":12,"_51":983,"_55":985},{"_43":982,"_30":45,"_46":47},"3efca932-e77b-436e-8a30-73aa9fe4d32e",{"_30":51,"_53":984},"/solutions/aiven-for-retail","Retail",{"_38":987,"_30":811,"_820":-5,"_821":988,"_828":829},"e96a94f5df1b",{"_30":40,"_41":989,"_48":49,"_11":12,"_50":12,"_51":991,"_55":993},{"_43":990,"_30":45,"_46":47},"7ddc00ac-3928-428c-8d61-d860d0011492",{"_30":51,"_53":992},"/solutions/aiven-for-software","Software",{"_38":995,"_30":811,"_820":-5,"_821":996,"_828":829},"31951636b02e3a0b1678f29e1347addc",{"_30":40,"_41":997,"_48":49,"_11":12,"_50":12,"_51":999,"_55":1001},{"_43":998,"_30":45,"_46":47},"8f78eb23-e1ba-46bf-b150-336732e1648e",{"_30":51,"_53":1000},"/solutions/aiven-for-fintech","Fintech",{"_38":1003,"_30":811,"_820":-5,"_821":1004,"_828":1010},"677cab651678",{"_30":40,"_41":1005,"_48":49,"_11":12,"_50":12,"_51":1007,"_55":1009},{"_43":1006,"_30":45,"_46":47},"be4378c0-8d66-4694-8a37-b9519f4b10ae",{"_30":51,"_53":1008},"/case-studies","View all case studies","highlight","By industry","Solutions",{"_38":1014,"_30":800,"_801":1015,"_951":-5,"_55":1095},"86ef58c39bfe",[1016,1043],{"_38":1017,"_30":805,"_806":807,"_796":1018,"_55":1042},"e43d5a594f23",[1019,1024,1029,1034,1039],{"_38":1020,"_30":811,"_820":-5,"_821":1021,"_828":829},"2036d8237b43",{"_30":40,"_226":1022,"_48":49,"_11":-5,"_50":12,"_51":-5,"_55":1023},"https://aiven.io/docs","Documentation",{"_38":1025,"_30":811,"_820":-5,"_821":1026,"_828":829},"ed594af59633",{"_30":40,"_226":1027,"_48":49,"_11":-5,"_50":12,"_51":-5,"_55":1028},"https://api.aiven.io/doc/","API reference",{"_38":1030,"_30":811,"_820":-5,"_821":1031,"_828":829},"c82f920659a8",{"_30":40,"_41":1032,"_48":49,"_11":12,"_50":12,"_51":1033,"_55":63},{"_43":60,"_30":45,"_46":47},{"_30":51,"_53":62},{"_38":1035,"_30":811,"_820":-5,"_821":1036,"_828":829},"7170cb9b5558",{"_30":40,"_41":1037,"_48":49,"_11":12,"_50":12,"_51":1038,"_55":223},{"_43":220,"_30":45,"_46":47},{"_30":51,"_53":222},{"_38":1040,"_30":811,"_820":-5,"_821":1041,"_828":829},"225c80591d03",{"_30":40,"_226":227,"_48":49,"_11":-5,"_50":12,"_51":-5,"_55":228},"For developers",{"_38":1044,"_30":805,"_806":856,"_796":1045,"_55":1094},"bd9cd65d4873",[1046,1054,1062,1070,1078,1086],{"_38":1047,"_30":811,"_820":-5,"_821":1048,"_828":829},"ec478f8daf01",{"_30":40,"_41":1049,"_48":49,"_11":12,"_50":12,"_51":1051,"_55":1053},{"_43":1050,"_30":45,"_46":47},"b72fddc9-a8e4-48f3-874d-2b0ad9c00992",{"_30":51,"_53":1052},"/tools/sql-syntax-checker","SQL Syntax Checker",{"_38":1055,"_30":811,"_820":-5,"_821":1056,"_828":829},"71f0ee5389c5",{"_30":40,"_41":1057,"_48":49,"_11":12,"_50":12,"_51":1059,"_55":1061},{"_43":1058,"_30":45,"_46":47},"cb4fe454-eec8-4099-9f4d-3b0e29f872b1",{"_30":51,"_53":1060},"/tools/sql-query-optimizer","SQL Optimizer",{"_38":1063,"_30":811,"_820":-5,"_821":1064,"_828":829},"e6d6398bc5dd",{"_30":40,"_41":1065,"_48":49,"_11":12,"_50":12,"_51":1067,"_55":1069},{"_43":1066,"_30":45,"_46":47},"b5993f84-5248-49e7-8e75-c6be58365e05",{"_30":51,"_53":1068},"/tools/pg-playground","PostgreSQL Playground",{"_38":1071,"_30":811,"_820":-5,"_821":1072,"_828":829},"0a61eb2e8e8d1b5e78583a538c897f1b",{"_30":40,"_41":1073,"_48":49,"_11":12,"_50":12,"_51":1075,"_55":1077},{"_43":1074,"_30":45,"_46":47},"a90fd771-37de-409a-b1ac-051a49883f80",{"_30":51,"_53":1076},"/tools/sql-formatter","SQL Formatter",{"_38":1079,"_30":811,"_820":-5,"_821":1080,"_828":829},"c587806c6549",{"_30":40,"_41":1081,"_48":49,"_11":12,"_50":12,"_51":1083,"_55":1085},{"_43":1082,"_30":45,"_46":47},"585a276e-c0b6-472f-8146-d0352ac67c5e",{"_30":51,"_53":1084},"/tools/kafka-visualization","Kafka Visualization",{"_38":1087,"_30":811,"_820":-5,"_821":1088,"_828":1010},"343dace9362f",{"_30":40,"_41":1089,"_48":49,"_11":12,"_50":12,"_51":1091,"_55":1093},{"_43":1090,"_30":45,"_46":47},"fae73ef3-8a98-4b39-96d3-33f1a3894e58",{"_30":51,"_53":1092},"/tools","View all free tools","Free tools","Developers",{"_38":1097,"_30":40,"_41":1098,"_48":49,"_11":12,"_50":12,"_51":1099,"_55":98},"0bebf54d384470d6b874e5458b0f0e36",{"_43":95,"_30":45,"_46":47},{"_30":51,"_53":97},"primaryCTA",{"_30":40,"_226":1102,"_48":49,"_11":-5,"_51":-5,"_55":1103},"https://console.aiven.io/signup","Start for free","secondaryCTA",{"_30":40,"_41":1106,"_48":49,"_11":12,"_51":1107,"_55":254},{"_43":251,"_30":45,"_46":47},{"_30":51,"_53":253},{"_30":51,"_53":1109},"main-menu","Main menu","primary","isPreview","sanitySecretForPreview","","isCI","isDarkTheme","isInklessTheme","routes/developer/layout",{"_1120":1121,"_1155":1156,"_1172":1173,"_1253":1254},"pageData",{"_1122":-5,"_1123":-5,"_1124":1125,"_1112":49,"_1153":1154},"query","queryParams","data",{"_16":12,"_17":1126,"_19":1127,"_21":1128,"_30":1129,"_32":1130,"_1131":-5,"_1132":-5,"_1133":1134,"_1135":-5,"_1136":-5,"_1137":-5,"_1138":1139,"_51":1149,"_1151":-5,"_55":1152},"2023-03-17T15:19:58Z","20f93b68-4c96-46eb-8900-a3b522bfa249","DEr13LRm8S6bXdr2YFSFN7","page","2023-06-21T07:58:39Z","content","infoBanner","infoBannerPosition","bottom","internalTags","modals","secondaryNavigation","seo",{"_30":1140,"_1141":49,"_1142":294,"_1143":1144,"_1147":1148},"metaData","excludeFromSitemap","metaDescription","metaImage",{"_30":814,"_817":1145},{"_43":1146,"_30":45},"image-6d1b413bef33c70c59b691abaf07da67a64f6ba1-1200x630-jpg","metaTitle","Aiven Developer Center | Your hub for data development ",{"_30":51,"_53":1150},"/developer","tags","Developer","availableLocales",[],"menu",[1157,1164,1168],{"_19":1158,"_1159":1160,"_51":563,"_9":1161,"_55":564,"_99":1160,"_1162":1163},"34643944-d559-485e-93a0-7d1f94a744e7","level",0,"/serve","children",[],{"_19":1165,"_1159":1160,"_51":551,"_9":1166,"_55":552,"_99":1160,"_1162":1167},"e6555a63-f0c2-406b-896c-cbe290019c3c","/store",[],{"_19":1169,"_1159":1160,"_51":547,"_9":1170,"_55":548,"_99":1160,"_1162":1171},"185b7847-7030-4530-ad9c-89a0772c8be9","/stream",[],"breadCrumbLabelDictionary",{"_1174":1175,"_1176":1152,"_1177":1178,"_1179":1180,"_547":548,"_1181":1182,"_1183":1184,"_1185":1186,"_1124":1187,"_563":564,"_1188":1189,"_1190":1191,"_1192":1193,"_1194":1195,"_1196":77,"_1197":1198,"_1199":1200,"_1201":1202,"_363":1203,"_1204":1205,"_1206":1207,"_1208":1209,"_1210":1211,"_326":1212,"_1213":1214,"_1215":1216,"_1217":1218,"_1219":985,"_1220":1221,"_1222":1223,"_1224":1225,"_1226":1227,"_1228":1229,"_321":1230,"_1231":1231,"_1232":1233,"_1234":1235,"_1236":1237,"_1238":1239,"_1240":1241,"_1242":1243,"_1244":63,"_551":552,"_1245":1246,"_1247":1248,"_1249":1250,"_1251":1252},"cloud-data-platform","Cloud data platform","developer","redis","Redis®","opensearch","OpenSearch®","aiven-api","Aiven API","roi","ROI","esg","ESG","Data","ai","AI","alloydb-omni","AlloyDB Omni","kafka","Apache Kafka®","machine-learning","Machine learning","sustainability","performance-optimization","Performance optimization","m3","M3","terraform","Terraform","Apache Flink®","guide","Guide","devops","DevOps","cluster-startup-program","Cluster startup program","byoc","BYOC","PostgreSQL®","hashicorp","HashiCorp","cost-optimization","Cost optimization","mysql","MySQL","retail","tutorial","Tutorial","dragonfly","Dragonfly","cassandra","Apache Cassandra®","clickhouse","ClickHouse®","integration","Integration","Grafana®","pgvector","real-time-analytics","Real-time analytics","valkey","Valkey","event-streaming","Event streaming","migration","Migration","netlify","Netlify","demo","Demo","open-source","cheat-sheet","Cheat sheet","aws","AWS","security","Security","gcp","GCP","technologyTags",[1255,1277,1289,1301,1313,1324,1334,1345,1357,1369,1381],{"_19":1256,"_30":1257,"_51":1258,"_55":1191,"_1259":-5,"_812":1260,"_806":1269,"_1275":172,"_1276":1160},"39fd5db7-c38d-4c87-8ce2-408bd8a79253","genericTag",{"_53":1190},"devCenterMetaDescription",{"_815":-5,"_817":1261},{"_1262":1263,"_1264":-5,"_1265":1266,"_1267":1268},"url","https://cdn.sanity.io/images/sczeoy4w/production/8371936a7906a98fd63ae43361a9157a4a1498e5-48x48.svg","altText","originalFilename","logo-alloydb-omni.svg","extension","svg",{"_19":1270,"_30":1271,"_51":1272,"_55":1274},"3340199e-976d-4da1-86fa-add893be0f5a","tagGroup",{"_53":1273},"technology","Technology","articleCount","resourceCount",{"_19":1278,"_30":1257,"_51":1279,"_55":1203,"_1259":1280,"_812":1281,"_806":1286,"_1275":1288,"_1276":229},"665d1689-e4a5-43d5-bfbe-a673d66b1a41",{"_53":363},"Move from batch processing to streaming while using SQL in the data pipeline | Discover the possibilities of Apache Flink® in the Aiven Developer Center",{"_815":-5,"_817":1282},{"_1262":1283,"_1264":1284,"_1265":1285,"_1267":1268},"https://cdn.sanity.io/images/sczeoy4w/production/c96d109662fa0afd283a3ce7fbd1af8a8005f682-64x64.svg","Aiven for Apache Flink® logo","logo-flink.svg",{"_19":1270,"_30":1271,"_51":1287,"_55":1274},{"_53":1273},6,{"_19":1290,"_30":1257,"_51":1291,"_55":1193,"_1259":1292,"_812":1293,"_806":1297,"_1275":1299,"_1276":1300},"3c547914-626c-4ae5-8d0c-a0400bcc5e7f",{"_53":1192},"Learn how to retroactively manage your data platforms with Terraform and much more | Discover the possibilities of Apache Kafka® in the Aiven Developer Center",{"_815":-5,"_817":1294},{"_1262":1295,"_1264":816,"_1265":1296,"_1267":1268},"https://cdn.sanity.io/images/sczeoy4w/production/184cb33e057478405636d30b72f56c94f88ca103-64x64.svg","logo-kafka.svg",{"_19":1270,"_30":1271,"_51":1298,"_55":1274},{"_53":1273},42,21,{"_19":1302,"_30":1257,"_51":1303,"_55":1227,"_1259":1304,"_812":1305,"_806":1310,"_1275":1312,"_1276":172},"94e23adc-203f-4482-b8df-63aeab555411",{"_53":1226},"Learn how to get ClickHouse® data into the Metabase Business Intelligence tool and get valuable insights into your data. Read more in the Aiven Developer Center",{"_815":-5,"_817":1306},{"_1262":1307,"_1264":1308,"_1265":1309,"_1267":1268},"https://cdn.sanity.io/images/sczeoy4w/production/41bf2f72c0a4c8def391b8bd40463dd894343541-64x64.svg","Aiven for Clickhouse logo","logo-clickhouse.svg",{"_19":1270,"_30":1271,"_51":1311,"_55":1274},{"_53":1273},8,{"_19":1314,"_30":1257,"_51":1315,"_55":1230,"_1259":1316,"_812":1317,"_806":1322,"_1275":100,"_1276":1160},"a40792b4-c5eb-4d81-9452-9a4bd322d700",{"_53":321},"Monitor your stack with a number of prebuilt dashboards for many services | Discover the possibilities of Aiven for Grafana® in our Developer Center",{"_815":-5,"_817":1318},{"_1262":1319,"_1264":1320,"_1265":1321,"_1267":1268},"https://cdn.sanity.io/images/sczeoy4w/production/6ae9b34c7ca56664b6e217e40c0b3e6948de81d3-64x64.svg","Aiven for Grafana® logo","logo-grafana.svg",{"_19":1270,"_30":1271,"_51":1323,"_55":1274},{"_53":1273},{"_19":1325,"_30":1257,"_51":1326,"_55":1200,"_1259":-5,"_812":1327,"_806":1332,"_1275":100,"_1276":1160},"5e22b8bd-9aba-4abc-84fe-60a512231fd9",{"_53":1199},{"_815":-5,"_817":1328},{"_1262":1329,"_1264":1330,"_1265":1331,"_1267":1268},"https://cdn.sanity.io/images/sczeoy4w/production/30db7a5b09c94a616a5ff1e52181f92b40fade73-64x64.svg","Aiven for M3","logo-m3.svg",{"_19":1270,"_30":1271,"_51":1333,"_55":1274},{"_53":1273},{"_19":1335,"_30":1257,"_51":1336,"_55":1218,"_1259":1337,"_812":1338,"_806":1343,"_1275":1288,"_1276":1312},"84cb2f1d-1db5-4eed-b049-86cc5f2715ef",{"_53":1217},"Learn how to safely migrate a MySQL® database to a new cloud provider and much more | Discover the possibilities of MySQL® in the Aiven Developer Center",{"_815":-5,"_817":1339},{"_1262":1340,"_1264":1341,"_1265":1342,"_1267":1268},"https://cdn.sanity.io/images/sczeoy4w/production/8443a7ce7f5ba06083c450ba0cce89a7151849b4-64x64.svg","Aiven for MySQL® logo","logo-mysql.svg",{"_19":1270,"_30":1271,"_51":1344,"_55":1274},{"_53":1273},{"_19":1346,"_30":1257,"_51":1347,"_55":1180,"_1259":1348,"_812":1349,"_806":1354,"_1275":1356,"_1276":273},"17773fab-43cf-4f70-873a-8da1274cde0f",{"_53":1179},"Implement OpenSearch® to analyse the data coming from Mastodon and more | Discover all its possibilities in the Aiven Developer Center and simplify your work",{"_815":-5,"_817":1350},{"_1262":1351,"_1264":1352,"_1265":1353,"_1267":1268},"https://cdn.sanity.io/images/sczeoy4w/production/24f8bc050ae61d964c383f4509c62c6cb865d6ef-64x64.svg","Aiven for OpenSearch® logo","logo-open-search.svg",{"_19":1270,"_30":1271,"_51":1355,"_55":1274},{"_53":1273},9,{"_19":1358,"_30":1257,"_51":1359,"_55":1212,"_1259":1360,"_812":1361,"_806":1365,"_1275":1367,"_1276":1368},"7585568a-42e1-4b7a-ba53-f416ed7c4ce2",{"_53":326},"Validate data as it goes into your databases to improve its quality and much more | Discover the possibilities of PostgreSQL® in the Aiven Developer Center",{"_815":-5,"_817":1362},{"_1262":1363,"_1264":861,"_1265":1364,"_1267":1268},"https://cdn.sanity.io/images/sczeoy4w/production/1e41c5eca58a19a523cfc3597c6ff0f3baf71ea4-64x64.svg","logo-postgresql.svg",{"_19":1270,"_30":1271,"_51":1366,"_55":1274},{"_53":1273},27,12,{"_19":1370,"_30":1257,"_51":1371,"_55":1178,"_1259":1372,"_812":1373,"_806":1378,"_1275":1380,"_1276":100},"170e8cc3-b016-45ee-a9db-efec32d13a04",{"_53":1177},"Add caching to your PostgreSQL® app | Make a Kubernetes-friendly Java app and more | Discover the possibilities of Redis® in the Aiven Developer Center",{"_815":-5,"_817":1374},{"_1262":1375,"_1264":1376,"_1265":1377,"_1267":1268},"https://cdn.sanity.io/images/sczeoy4w/production/c49aa024b64012920f36be76dce8581e6a340388-64x64.svg","Aiven for Redis® logo","logo-redis.svg",{"_19":1270,"_30":1271,"_51":1379,"_55":1274},{"_53":1273},5,{"_19":1382,"_30":1257,"_51":1383,"_55":1235,"_1259":-5,"_812":1384,"_806":1389,"_1275":1380,"_1276":1160},"d066802f-72f8-49bf-93a7-3c4196a99892",{"_53":1234},{"_815":-5,"_817":1385},{"_1262":1386,"_1264":1387,"_1265":1388,"_1267":1268},"https://cdn.sanity.io/images/sczeoy4w/production/f997cab5874d12c680040053b0a516337323f549-48x48.svg","Aiven for Valkey","logo-valkey.svg",{"_19":1270,"_30":1271,"_51":1390,"_55":1274},{"_53":1273},"routes/developer/$",{"_1120":1393,"_1675":-5,"_1676":1677,"_1688":1689,"_1692":1693},{"_1122":-5,"_1123":-5,"_1124":1394,"_1112":49,"_1153":1674},{"_16":12,"_19":1395,"_30":1396,"_32":1397,"_1398":1399,"_1409":1660,"_1411":1661,"_1417":1663,"_1135":-5,"_1424":1665,"_1138":-5,"_51":1666,"_1433":1667,"_1151":1668,"_55":1673},"3f924711-f70e-4091-87da-9e4fdc57cc4e","devCollection","2024-09-16T15:12:15Z","articleItems",[1400,1442,1470,1498,1523,1551,1576,1601,1633],{"_16":12,"_17":1401,"_19":1402,"_21":1403,"_23":1404,"_30":1407,"_32":1408,"_1409":1410,"_1411":1412,"_1417":1418,"_1135":1419,"_1424":1425,"_1138":1426,"_51":1431,"_1433":1434,"_1151":1435,"_55":1441},"2023-08-25T09:51:41Z","a21b32f0-aaab-4720-8976-c439d5a3f4c0","g8eHHsLV7ci6mnlkKzoL5m",{"_25":1405},{"_27":1402,"_28":1406},"VeNKqGNmt13PzvGhqsStnY","devArticle","2025-09-30T03:45:23Z","body","This article shows the Terraform configuration for setting up an [Aiven for Apache Kafka®](https://aiven.io/kafka) service, configuring a topic, and adding a Kafka Connector to send the data from the topic over HTTP using the [HTTP sink connector]. This is a great way to use webhooks or HTTP requests as a generic connector to send the data to\nanother platform.\n\nThe setup needs the Aiven for Apache Kafka service and the Kafka Connect\nservice, plus a service integration to connect the two. The overall setup looks something like the diagram below:\n\n\n:::mermaid\n\n flowchart LR\n subgraph k1 [Kafka cluster]\n topic[Topic]\n end\n sikafka{{Service Integration}}\n subgraph kc1 [Kafka Connect]\n connector[HTTP sink connector]\n end\n webhook([HTTP destination])\n topic-->sikafka-->connector-->webhook\n:::\n\nThe Aiven for Apache Kafka and Kafka Connect services are connected with a service integration. The Kafka Connect service has the HTTP sink configured and this connects the topic to the HTTP destination. The HTTP destination is an external location defined by ``http.url`` in the HTTP sink connector\nconfiguration.\n\n## Define the setup\n\nBe sure to check out the [getting started guide](https://aiven.io/docs/tools/terraform/get-started) to\nlearn about the common files required to execute the following recipe. For example, you'll need to declare the variables for ``project_name`` and\n``api_token``.\n\n\n## Common files \n\nNavigate to a new folder and add the following files.\n\nAdd the following to a new ``provider.tf`` file:\n\n```toml\nterraform {\n required_providers {\n aiven = {\n source = \"aiven/aiven\"\n version = \">=4.0.0, < 5.0.0\"\n }\n }\n}\n\nprovider \"aiven\" {\n api_token = var.aiven_api_token\n}\n```\n\nYou can also set the environment variable ``TF_VAR_aiven_api_token`` for the ``api_token`` property. With this, you don't need to pass the ``-var-file`` flag when executing Terraform commands.\n\nTo avoid including sensitive information in source control, the variables are defined here in the ``variables.tf`` file. You can then use a ``*.tfvars`` file with the actual values so that Terraform receives the values during runtime, and exclude it.\n\nThe ``variables.tf`` file defines the API token, the project name to use, and the prefix for the service name:\n\n```toml\nvariable \"aiven_api_token\" {\n description = \"Aiven console API token\"\n type = string\n}\n\nvariable \"project_name\" {\n description = \"Aiven console project name\"\n type = string\n}\n```\n\nThe ``var-values.tfvars`` file holds the actual values and is passed to Terraform using the ``-var-file=`` flag.\n\n``var-values.tfvars`` file:\n\n```\naiven_api_token = \"\"\nproject_name = \"\"\n```\n\n## Services.tf file\n\nThe sample Terraform file to create and connect all the services is shown below. This file uses sample data; comments are added to indicate settings that are likely to need changing to suit your use case.\n\n``services.tf`` file:\n\n```toml\n# Kafka service\nresource \"aiven_kafka\" \"project_kafka\" {\n project = var.project_name # from variables.tf and supplied at run time\n cloud_name = \"google-europe-west1\"\n plan = \"business-4\"\n service_name = \"my-kafka-demo\"\n kafka_user_config {\n kafka_version = \"3.2\"\n kafka_rest = true\n kafka {\n auto_create_topics_enable = true\n }\n }\n}\n\n# Kafka topic, in the cluster defined above\nresource \"aiven_kafka_topic\" \"user_activity\" {\n project = var.project_name\n service_name = aiven_kafka.project_kafka.service_name\n topic_name = \"user_activity\"\n partitions = 3\n replication = 2\n}\n\n# Kafka Connect service\nresource \"aiven_kafka_connect\" \"data_connector\" {\n project = var.project_name\n cloud_name = \"google-europe-west1\"\n plan = \"business-4\"\n service_name = \"my-kafka-demo-connector\"\n}\n\n# Integration between kafka and kafka connect\nresource \"aiven_service_integration\" \"kafka_to_data_connector\" {\n project = var.project_name\n integration_type = \"kafka_connect\"\n source_service_name = aiven_kafka.project_kafka.service_name\n destination_service_name = aiven_kafka_connect.data_connector.service_name\n}\n\n# Kafka connector: this one is an HTTP sink\nresource \"aiven_kafka_connector\" \"kafka_webhook_sink\" {\n project = var.project_name\n service_name = aiven_kafka_connect.data_connector.service_name\n connector_name = \"my-http-sink\"\n config = {\n # Which topic (or topics) should the data come from?\n \"topics\" = aiven_kafka_topic.user_activity.topic_name\n \"connector.class\" = \"io.aiven.kafka.connect.http.HttpSinkConnector\"\n \"name\" = \"my-http-sink\"\n # Edit where the HTTP data should be sent\n \"http.url\" = \"https://example.com/endpoint\"\n \"http.authorization.type\" = \"none\"\n \"http.headers.content.type\" = \"application/json\"\n \"key.converter\" = \"org.apache.kafka.connect.storage.StringConverter\"\n \"value.converter\" = \"org.apache.kafka.connect.storage.StringConverter\"\n }\n\n # Make sure that the connect service is ready before creating this\n depends_on = [\n aiven_kafka_connect.data_connector,\n aiven_service_integration.kafka_to_data_connector\n ]\n}\n```\n\nThis example creates two Aiven services: one Aiven for Apache Kafka service, and a Kafka Connect service. It adds a service integration so that the the connectors can access the data in the Kafka cluster. There is a topic defined ``user_activity``, and this is referred to by the Kafka connector configuration as the source of the data to send over HTTP. The connector also defines where the HTTP requests should be sent to, using the ``http.url`` setting.\n\nTo avoid any race conditions in this setup, the ``depends_on`` clause makes sure that the connector will only be configured when both the Kafka Connect service and its integration to Kafka are in place.\n\n## Execute the files \n\nThe ``init`` command performs several different initialization steps in order to prepare the current working directory for use with Terraform. In our case, this command automatically finds, downloads, and installs the necessary Aiven Terraform provider plugins.\n \n```bash\nterraform init\n```\n\nThe ``plan`` command creates an execution plan and shows you the resources that will be created (or modified) for you. This command does not actually create any resource; this is more like a preview.\n\n```bash\nterraform plan -var-file=var-values.tfvars\n```\n\nIf you're satisfied with the output of ``terraform plan``, go ahead and run the ``terraform apply`` command which actually does the task or creating (or modifying) your infrastructure resources. \n\n```bash\nterraform apply -var-file=var-values.tfvars\n```\n \nTry out this recipe by defining an HTTP endpoint where you can receive and acknowledge HTTP requests in the connector configuration. Then produce some data to the ``user_activity`` topic (any JSON data is fine), and observe that this is then sent over HTTP.\n\nYou could use this setup for relaying payloads to platforms that don't have specific connectors available. The HTTP sink connector is also an excellent tool for integrating with webhook-ready platforms like functions-as-a-service ([Amazon Lambda](https://aws.amazon.com/lambda/), [Cloudflare\nWorkers](https://workers.cloudflare.com/) or [Zapier](https://zapier.com/).\n\n\n## Further resources\n\nHere are some resources with additional information, examples and documentation for working with the technologies in this recipe:\n\n- [Configuration options for Kafka]\n- [HTTP sink documentation and examples]\n- [Webhook data from Apache Kafka®️ to the world: using the HTTP sink connector](https://aiven.io/developer/kafka-webhooks-http-sink)\n","feature",{"_30":1413,"_1414":49,"_1415":1416},"featuredItem","active","endDate","2023-09-01T09:51:38.951Z","hierarchy",{"_43":1169,"_30":45},[1420],{"_38":1421,"_43":1422,"_30":1423},"bfa033e687c3","3d8ecd72-b886-4aaa-b207-2fbafc931f22","tag","publishedAt","2023-08-25T07:49:18.951Z",{"_30":1140,"_1143":1427,"_1147":1430},{"_30":814,"_817":1428},{"_43":1429,"_30":45},"image-1007d93722cce4ccff54737915530cdbf284f599-1567x844-png","Set Up Kafka Topics with HTTP Sink via Terraform",{"_30":51,"_53":1432},"/developer/kafka-with-http-sink","subtitle","Use an HTTP sink connector to feed data into Apache Kafka® and deploy it using Terraform. A part of Aiven's Terraform Cookbook.",[1436,1438],{"_38":1437,"_43":1290,"_30":1423},"d2700d8d9f7f",{"_38":1439,"_43":1440,"_30":1423},"249c89177c50","6496a8e4-6fbe-43d6-b945-f2505fd3b34a","Configure Apache Kafka® with topics and an HTTP sink connector using Terraform",{"_16":12,"_17":1443,"_19":1444,"_21":1445,"_30":1407,"_32":1446,"_1409":1447,"_1411":1448,"_1417":1450,"_1135":1451,"_1424":1454,"_1138":1455,"_51":1459,"_1433":1461,"_1151":1462,"_55":1469},"2023-08-23T09:13:28Z","75e9857b-65bd-4036-9bab-257f0ddbfb8c","VeNKqGNmt13PzvGhqsShEs","2024-01-29T15:45:51Z","This article shows by way of example how to integrate an [Aiven for Apache Kafka®](https://aiven.io/kafka) service with a [Aiven for ClickHouse®](https://aiven.io/clickhouse) database using [Terraform provider for Aiven](https://registry.terraform.io/providers/aiven/aiven/latest/docs). An Apache Kafka® source topic is used as a data source and Aiven for ClickHouse® is used to filter or transform the raw data with a materialized view before writing it to a regular table.\n\n\nFirst, check out how sensor data can look like for a better understanding of this recipe and the ``clickhouse_kafka_user_config`` Terraform block used in this article.\n\n```json\n{\n\"sensor_id\": 10000001,\n\"ts\": \"2022-12-01T10:08:24.446369\",\n\"key\": \"cpu_usage\",\n\"value\": 96\n}\n```\n\n## Describe the setup \n\n:::mermaid\n flowchart LR\n id1(iot_measurements_topic)\n id2[(iot_measurements)]\n id3(edge_measurements_raw_table)\n id4(cpu_high_usage_table)\n subgraph Aiven for Apache Kafka\n id1\n end\n subgraph Aiven for ClickHouse\n id2\n subgraph iot_measurements DB\n id3-->|Filter|id4\n end\n end\n id1-->|Service integration|id2\n:::\n\nImagine that you've been collecting IoT measurements from thousands of sensors and these metrics are being populated in an Apache Kafka topic called ``iot_measurements``. Now, you'd like to set up an Aiven for ClickHouse database and write filtered messages into table ``cpu_high_usage``.\n\nThis recipe calls for the following:\n\n1. Set up an Aiven for ClickHouse database for writing and processing raw data. [Sign up for an Aiven account]( https://console.aiven.io/signup) to do so.\n2. Insert the measurements data from Apache Kafka topic ``iot_measurements`` into the Aiven for ClickHouse database.\n3. Filter the data and save the output to the new ``cpu_high_usage`` table.\n\n\n## Common files \n\nNavigate to a new folder and add the following files:\n\n1. ``provider.tf`` file\n\n```toml\nterraform {\n required_providers {\n aiven = {\n source = \"aiven/aiven\"\n version = \">=4.0.0, < 5.0.0\"\n }\n }\n}\n\nprovider \"aiven\" {\n api_token = var.aiven_api_token\n}\n```\n\nYou can set environment variable ``TF_VAR_aiven_api_token`` for the ``api_token`` property so that you don't need to pass the ``-var-file`` flag when executing Terraform commands.\n\n``variables.tf`` file\n\nUse it for defining the variables to avoid including sensitive information in source control. The ``variables.tf`` file defines the API token, the project name, and the prefix for the service name.\n\n```toml\nvariable \"aiven_api_token\" {\n description = \"Aiven console API token\"\n type = string\n}\n\nvariable \"project_name\" {\n description = \"Aiven console project name\"\n type = string\n}\n```\n\n``*.tfvars`` file\n\nUse it to indicate the actual values of the variables so that they can be passed (with the ``-var-file=`` flag) to Terraform during runtime and excluded later on. Configure the ``var-values.tfvars`` file as follows:\n\n```toml\naiven_api_token = \"\"\nproject_name = \"\"\n```\n\n## Services.tf file \n\nThe following Terraform script initializes both Aiven for Apache Kafka and Aiven for ClickHouse services, creates the service integration, the source Apache Kafka topic, and the Aiven for ClickHouse database.\n\n```toml\nresource \"aiven_kafka\" \"kafka\" {\n project = var.project_name\n cloud_name = \"google-europe-west1\"\n plan = \"business-4\"\n service_name = \"kafka-gcp-eu\"\n maintenance_window_dow = \"monday\"\n maintenance_window_time = \"10:00:00\"\n}\n\nresource \"aiven_kafka_topic\" \"source\" {\n project = var.project_name\n service_name = aiven_kafka.kafka.service_name\n partitions = 50\n replication = 3\n topic_name = \"iot_measurements\"\n}\n\nresource \"aiven_clickhouse\" \"clickhouse\" {\n project = var.project_name\n cloud_name = \"google-europe-west1\"\n plan = \"startup-8\"\n service_name = \"clickhouse-gcp-eu\"\n maintenance_window_dow = \"monday\"\n maintenance_window_time = \"10:00:00\"\n}\n\nresource \"aiven_service_integration\" \"clickhouse_kafka_source\" {\n project = var.project_name\n integration_type = \"clickhouse_kafka\"\n source_service_name = aiven_kafka.kafka.service_name\n destination_service_name = aiven_clickhouse.clickhouse.service_name\n clickhouse_kafka_user_config {\n tables {\n name = \"edge_measurements_raw\"\n group_name = \"clickhouse-ingestion\"\n data_format = \"JSONEachRow\"\n columns {\n name = \"sensor_id\"\n type = \"UInt64\"\n }\n columns {\n name = \"ts\"\n type = \"DateTime64(6)\"\n }\n columns {\n name = \"key\"\n type = \"LowCardinality(String)\"\n }\n columns {\n name = \"value\"\n type = \"Float64\"\n }\n topics {\n name = aiven_kafka_topic.source.topic_name\n }\n }\n }\n}\n\nresource \"aiven_clickhouse_database\" \"measurements\" {\n project = var.project_name\n service_name = aiven_clickhouse.clickhouse.service_name\n name = \"iot_measurements\"\n}\n```\n\n## Execute the files \n\nRun the following command:\n\n```bash\nterraform init\n```\n \nThe ``init`` command performs initialization operations to prepare the working directory for use with Terraform. For this recipe, ``init`` automatically finds, downloads, and installs the necessary Aiven Terraform Provider plugins.\n\nRun the following command:\n\n```bash\nterraform plan -var-file=var-values.tfvars\n```\n\nThe ``plan`` command creates an execution plan and shows the resources to be created (or modified). This command doesn't actually create any resources but gives you a heads-up on what's going to happen next.\n\nIf the output of ``terraform plan`` looks as expected, run the following command:\n\n```bash\nterraform apply -var-file=var-values.tfvars\n```\n \nThe ``terraform apply`` command creates (or modifies) your infrastructure resources.\n\n## Check out the results\n\n* Resource ``aiven_clickhouse`` creates an Aiven for ClickHouse service with the project name, the cloud name (provider, region, zone), the Aiven service plan, and the service name as specified in the ``services.tf`` file.\n* Resource ``aiven_clickhouse_database`` creates a database that can be used to further transform the ingested data and perform analytics on it.\n* Resource ``aiven_kafka`` creates an Aiven for Apache Kafka cluster.\n* Resource ``aiven_kafka_topic`` creates Apache Kafka topic ``iot_measurements``.\n* Resource ``aiven_service_integration`` creates the integration between the Aiven for Apache Kafka and the Aiven for ClickHouse service.\n\nThe service integration creates a database to insert the ingested data to. In this instance, the database name is ``service_kafka-gcp-eu`` (it depends on the Kafka service name) and the table name is ``edge_measurements_raw`` as specified in the code.\n\n## Learn more\n\nWhen you use this recipe, parameters and configurations will vary from those used in this article. For Aiven for Apache Kafka and Aiven for ClickHouse advanced parameters, a related blog, and instructions on how to get started with Aiven Terraform Provider, see [Set up your first Aiven Terraform project](https://aiven.io/docs/tools/terraform/get-started.html).\n\n* You can [create databases and tables](https://aiven.io/docs/products/clickhouse/howto/integrate-kafka.html#update-apache-kafka-integration-settings) so that you can [read and store your data](https://aiven.io/docs/products/clickhouse/howto/integrate-kafka.html#read-and-store-data).\n* You can also [create a materialized view](https://aiven.io/docs/products/clickhouse/howto/materialized-views.html) to store the Kafka® messages in Aiven for ClickHouse.\n* [How to send and receive application data from Apache Kafka® using Apache Avro™](https://aiven.io/developer/how-to-send-and-receive-application-data-from-apache-kafka)",{"_30":1413,"_1414":49,"_1415":1449},"2023-08-30T09:09:04.401Z",{"_43":1169,"_30":45},[1452],{"_38":1453,"_43":1422,"_30":1423},"4c2712ce23ea","2023-08-23T07:48:51.992Z",{"_30":1140,"_1143":1456},{"_30":814,"_817":1457},{"_43":1458,"_30":45},"image-9c0543162ffd6af1c019f6e8937c6770b21318e4-1567x844-png",{"_30":51,"_53":1460},"/developer/kafka-source-for-clickhouse","Deploy an Aiven for Apache Kafka® node and integrate it as a data source with Aiven for ClickHouse® using Terraform. A part of Aiven's Terraform Cookbook.",[1463,1465,1467],{"_38":1464,"_43":1290,"_30":1423},"9b6cad2a1b3c",{"_38":1466,"_43":1440,"_30":1423},"e127fafb52fe",{"_38":1468,"_43":1302,"_30":1423},"901e5faf2d4a","Aiven for Apache Kafka® as a source for Aiven for ClickHouse®",{"_16":12,"_17":1471,"_19":1472,"_21":1473,"_30":1407,"_32":1474,"_1409":1475,"_1411":1476,"_1417":1478,"_1135":1479,"_1424":1482,"_1138":1483,"_51":1487,"_1433":1489,"_1151":1490,"_55":1497},"2023-08-22T08:05:04Z","373999f2-cb1a-48f5-8bba-b8a6295ba885","VeNKqGNmt13PzvGhqsSeIQ","2024-01-29T15:45:36Z","The [Aiven Terraform Provider](https://registry.terraform.io/providers/aiven/aiven/latest/docs) is a great choice for provisioning an [Aiven for Apache Kafka®](https://aiven.io/kafka) cluster with Kafka Connect enabled and the [Debezium source connector for PostgreSQL®](https://aiven.io/docs/products/kafka/kafka-connect/howto/debezium-source-connector-pg.html) configured.\n\nLet's check out the following diagram to understand the setup.\n\n:::mermaid\n\n flowchart LR\n id1[(PostgreSQL service on GCP)]\n subgraph Kafka Connect\n Debezium-Source-Connector-PG\n end\n subgraph Apache Kafka on Azure\n Topic\n end\n id1 --->|wal2json plugin| Debezium-Source-Connector-PG --->|publishes changes| Topic\n\n:::\n\n## Describe the setup\n\nThis terraform recipe will provision one [Aiven for PostgreSQL®](https://aiven.io/postgresql) database service, one Aiven for Apache Kafka service, and a separate Aiven for Apache Kafka Connect service with a Debezium source connector for PostgreSQL enabled and configured to connect to the PostgreSQL database and capture any changes in tables. The Aiven for Apache Kafka service is deployed in the Azure cloud, whereas the PostgreSQL database, like the Aiven for Apache Kafka Connector service, is deployed in the Google Cloud.\n\nAiven makes it very easy to configure services in different clouds that integrate seamlessly. As soon as any of the monitored tables is inserted or updated with new data, the Debezium connector will capture the data change and convert table data into a JSON payload and produce messages to the relevant Kafka topic. Some of these services are created on one cloud provider and some on another cloud provider, to demonstrate how easy it is with Aiven to integrate services across multiple cloud vendors.\n\n::alert[Aiven provides the option to run Kafka Connect on the same nodes as your Kafka cluster, sharing the resources. This is a low-cost way to get started with Kafka Connect. A standalone Aiven for Apache Kafka® Connect allows you to scale independently, offers more CPU time and memory for the Kafka Connect service and reduces load on nodes, making the cluster more stable.]{title=\"Warning\"}\n\nBe sure to check out the [getting started guide](https://aiven.io/docs/tools/terraform/get-started) to learn about the common files required to execute the following recipe.\n\nFor example, you'll need to declare the variables for ``project`` and ``api_token``.\n\n## Common files \n\nNavigate to a new folder and add the following files.\n\nAdd the following to a new ``provider.tf`` file:\n\n```toml\nterraform {\n required_providers {\n aiven = {\n source = \"aiven/aiven\"\n version = \">=4.0.0, < 5.0.0\"\n }\n }\n}\n\nprovider \"aiven\" {\n api_token = var.aiven_api_token\n}\n```\n\nYou can also set the environment variable ``TF_VAR_aiven_api_token`` for the ``api_token`` property. With this, you don't need to pass the ``-var-file`` flag when executing Terraform commands.\n\nTo avoid including sensitive information in source control, the variables are defined here in the ``variables.tf`` file. You can then use a ``*.tfvars`` file with the actual values so that Terraform receives the values during runtime, and exclude it.\n\nThe ``variables.tf`` file defines the API token, the project name to use, and the prefix for the service name:\n\n```toml\nvariable \"aiven_api_token\" {\ndescription = \"Aiven console API token\"\ntype = string\n}\n\nvariable \"project_name\" {\ndescription = \"Aiven console project name\"\ntype = string\n}\n```\n\nThe ``var-values.tfvars`` file holds the actual values and is passed to Terraform using the ``-var-file=`` flag.\n\n``var-values.tfvars`` file:\n\n```toml\naiven_api_token = \"\"\nproject_name = \"\"\n```\n\n## Services.tf file\n\nThe ``services.tf`` file for the provisioning of these three services, service integration, and related resource is this:\n\n```toml\nresource \"aiven_pg\" \"demo-pg\" {\n project = var.project_name\n service_name = \"demo-postgres\"\n cloud_name = \"google-europe-north1\"\n plan = \"business-4\"\n}\n\nresource \"aiven_kafka\" \"demo-kafka\" {\n project = var.project_name\n cloud_name = \"google-europe-north1\"\n plan = \"business-4\"\n service_name = \"demo-kafka\"\n maintenance_window_dow = \"saturday\"\n maintenance_window_time = \"10:00:00\"\n kafka_user_config {\n kafka_rest = true\n kafka_connect = false\n schema_registry = true\n kafka_version = \"3.4\"\n\n kafka {\n auto_create_topics_enable = true\n num_partitions = 3\n default_replication_factor = 2\n min_insync_replicas = 2\n }\n\n kafka_authentication_methods {\n certificate = true\n }\n\n }\n}\n\nresource \"aiven_kafka_connect\" \"demo-kafka-connect\" {\n project = var.project_name\n cloud_name = \"google-europe-north1\"\n plan = \"business-4\"\n service_name = \"demo-kafka-connect\"\n maintenance_window_dow = \"monday\"\n maintenance_window_time = \"10:00:00\"\n\n kafka_connect_user_config {\n kafka_connect {\n consumer_isolation_level = \"read_committed\"\n }\n\n public_access {\n kafka_connect = false\n }\n }\n}\n\nresource \"aiven_service_integration\" \"i1\" {\n project = var.project_name\n integration_type = \"kafka_connect\"\n source_service_name = aiven_kafka.demo-kafka.service_name\n destination_service_name = aiven_kafka_connect.demo-kafka-connect.service_name\n\n kafka_connect_user_config {\n kafka_connect {\n group_id = \"connect\"\n status_storage_topic = \"__connect_status\"\n offset_storage_topic = \"__connect_offsets\"\n }\n }\n}\n\nresource \"aiven_kafka_connector\" \"kafka-pg-source\" {\n project = var.project_name\n service_name = aiven_kafka_connect.demo-kafka-connect.service_name\n connector_name = \"kafka-pg-source\"\n\n config = {\n \"name\" = \"kafka-pg-source\"\n \"connector.class\" = \"io.debezium.connector.postgresql.PostgresConnector\"\n \"snapshot.mode\" = \"initial\"\n \"database.hostname\" = sensitive(aiven_pg.demo-pg.service_host)\n \"database.port\" = sensitive(aiven_pg.demo-pg.service_port)\n \"database.password\" = sensitive(aiven_pg.demo-pg.service_password)\n \"database.user\" = sensitive(aiven_pg.demo-pg.service_username)\n \"database.dbname\" = \"defaultdb\"\n \"database.server.name\" = \"replicator\"\n \"database.ssl.mode\" = \"require\"\n \"include.schema.changes\" = true\n \"include.query\" = true\n \"table.include.list\" = \"public.tab1\"\n \"plugin.name\" = \"pgoutput\"\n \"publication.autocreate.mode\" = \"filtered\"\n \"decimal.handling.mode\" = \"double\"\n \"_aiven.restart.on.failure\" = \"true\"\n \"heartbeat.interval.ms\" = 30000\n \"heartbeat.action.query\" = \"INSERT INTO heartbeat (status) VALUES (1)\"\n }\n depends_on = [aiven_service_integration.i1]\n}\n```\n\n\n## Execute the files \n\nThe ``init`` command performs several different initialization steps in order to prepare the current working directory for use with Terraform. In our case, this command automatically finds, downloads, and installs the necessary Aiven Terraform provider plugins.\n \n```shell\nterraform init\n```\n\nThe ``plan`` command creates an execution plan and shows you the resources that will be created (or modified) for you. This command does not actually create any resource; this is more like a preview.\n\n```bash\nterraform plan -var-file=var-values.tfvars\n```\n\nIf you're satisfied with the output of ``terraform plan``, go ahead and run the ``terraform apply`` command which actually does the task or creating (or modifying) your infrastructure resources. \n\n```bash\nterraform apply -var-file=var-values.tfvars\n```\n\nLet's go over a few of these configurations and understand their functions:\n\n- The ``auto_create_topics_enable = true`` property allows the Debezium connector to send messages to a non-existing topic.\n- The ``kafka_connect = false`` property is used because we want to create a separate Aiven for Apache Kafka Connect service.\n- The resource ``aiven_service_integration.i1`` configures the integration between the Aiven for Apache Kafka service and the Aiven for Apache Kafka Connect service. This integration uses two internal topics for storing status and offset.\n- ``group_id`` under ``kafka_connect_user_config`` is a unique ID that identifies the Kafka Connect cluster.\n- ``status_storage_topic`` and ``offset_storage_topic`` identify the name of the internal Kafka topics that store the connector status and the connector offsets respectively.\n- `snapshot.mode = \"initial\"` means the connector will, upon first connection, snapshot the entire structure and data of the linked database. After this snapshot, it will then capture and stream row-level changes. This initial capture ensures no data is lost in transition, but it might put some load on the database when first activated.\n- `plugin.name = \"pgoutput\"` specifies that the replication output plugin used by Debezium for PostgreSQL is “pgoutput.” It’s the standard logical decoding output plugin in recent PostgreSQL versions and is supported natively without needing additional extensions.\n- `publication.autocreate.mode = \"filtered\"` means that the connector will either use an existing logical replication publication with the specified name or create one if it doesn’t exist. The “filtered” mode ensures that only the changes to the tables specified in the configuration (like \"table.include.list\") are included in the publication.\n- The Debezium source connector for PostgreSQL listens for all data changes on one or more tables, including schema changes. In our case, the table that is monitored for any data change is \"tab1\" in ``defaultdb`` database under ``public`` schema. The plugin used to capture changes is ``wal2json`` that converts WAL events (WAL stands for Write Ahead Logging) into JSON payload that is sent to the Kafka topic via the Kafka connect service. The Kafka topic that the Debezium connector creates has the name ``replicator.public.tab1``, where \"replicator\" is the logical database used by Debezium connector to monitor for data changes and \"public\" and \"tab1\" are the name of the PostgreSQL schema and table name respectively. \n- The ``depends_on`` property establishes a dependency between the services creation in order to avoid failures.\n\n## More resources\n\nKeep in mind that some parameters and configurations will vary for your case. A reference to some of the advanced Apache Kafka configurations and other related resources:\n\n- [List of advanced Apache Kafka configurations](https://aiven.io/docs/products/kafka/kafka-connect/reference/advanced-params.html)\n- [Create a Debezium source connector](https://aiven.io/docs/products/kafka/kafka-connect/howto/debezium-source-connector-pg.html)\n- [List of available Apache Kafka® Connect connectors](https://aiven.io/docs/products/kafka/kafka-connect/concepts/list-of-connector-plugins.html)",{"_30":1413,"_1414":49,"_1415":1477},"2023-08-29T08:05:01.401Z",{"_43":1169,"_30":45},[1480],{"_38":1481,"_43":1422,"_30":1423},"da08866f08b2","2023-08-21T08:28:37.110Z",{"_30":1140,"_1143":1484},{"_30":814,"_817":1485},{"_43":1486,"_30":45},"image-91e0c19ac4bc1d6c43e69f625cc17d1b5242f954-1567x844-png",{"_30":51,"_53":1488},"/developer/debezium-source-postgresql-kafka-across-clouds","Use Debezium as a source connector to integrate PostgreSQL® and Apache Kafka® and use Terraform to deploy. A part of Aiven's Terraform Cookbook.",[1491,1493,1495],{"_38":1492,"_43":1290,"_30":1423},"22dc7d464d47",{"_38":1494,"_43":1440,"_30":1423},"698a207441ef",{"_38":1496,"_43":1358,"_30":1423},"c23020a86c2b","Debezium source connector - PostgreSQL® to Apache Kafka®",{"_16":12,"_17":1499,"_19":1500,"_21":1501,"_30":1407,"_32":1502,"_1409":1503,"_1411":1504,"_1417":1506,"_1135":1507,"_1424":1482,"_1138":1510,"_51":1514,"_1433":1516,"_1151":1517,"_55":1522},"2023-08-22T07:57:28Z","55e10e8d-4c91-4fe0-97c4-f92b09db8840","VeNKqGNmt13PzvGhqsSesG","2024-01-29T15:45:42Z","The [Aiven Terraform Provider](https://registry.terraform.io/providers/aiven/aiven/latest/docs) is a great choice for provisioning an [Aiven for Apache Kafka®](https://aiven.io/kafka) cluster with Kafka Connect enabled and the [MongoDB source connector](https://aiven.io/docs/products/kafka/kafka-connect/howto/mongodb-poll-source-connector.html) configured.\n\nLet's check out the following diagram to understand the setup.\n\n:::mermaid\n\n flowchart LR\n id1[(MongoDB Database)]\n subgraph Kafka Connect\n MongoDB-Source-Connector\n end\n subgraph Apache Kafka\n Topic\n end\n id1 --->|polls changes| MongoDB-Source-Connector -->|publishes changes| Topic\n\n:::\n\n## Describe the setup\n\nHere is the Terraform recipe that will spin up an Aiven for Apache Kafka service with Kafka Connect enabled. This recipe will also create and configure a MongoDB source connector.\n\n::alert[Aiven provides the option to run Kafka Connect on the same nodes as your Kafka cluster, sharing the resources. This is a low-cost way to get started with Kafka Connect. A standalone Aiven for Apache Kafka® Connect allows you to scale independently, offers more CPU time and memory for the Kafka Connect service and reduces load on nodes, making the cluster more stable.]{title=\"Warning\"} \n\nBefore you begin, you will require a MongoDB database and the related database connection information. You'll also need to make sure that the database is reachable from the public internet (unless it's part of a paired VPC).\n\nBe sure to check out the [getting started guide](https://aiven.io/docs/tools/terraform/get-started) to learn about the common files required to execute the following recipe. For example, you'll need to declare the variables for ``project_name``, ``api_token``, ``service_name_prefix``, and ``mongodb_connection_uri``.\n\n## Common files\n\n\nNavigate to a new folder and add the following files.\n\nAdd the following to a new ``provider.tf`` file:\n\n```toml\nterraform {\n required_providers {\n aiven = {\n source = \"aiven/aiven\"\n version = \">=4.0.0, < 5.0.0\"\n }\n }\n}\n\nprovider \"aiven\" {\n api_token = var.aiven_api_token\n}\n```\n\nYou can also set the environment variable ``TF_VAR_aiven_api_token`` for the ``api_token`` property. With this, you don't need to pass the ``-var-file`` flag when executing Terraform commands.\n\nTo avoid including sensitive information in source control, the variables are defined here in the ``variables.tf`` file. You can then use a ``*.tfvars`` file with the actual values so that Terraform receives the values during runtime, and exclude it.\n\nThe ``variables.tf`` file defines the API token, the project name to use, and the prefix for the service name:\n\n```toml\nvariable \"aiven_api_token\" {\n description = \"Aiven console API token\"\n type = string\n}\n\nvariable \"project_name\" {\n description = \"Aiven console project name\"\n type = string\n}\n\nvariable \"service_name_prefix\" {\n description = \"A string to prepend to the service name\"\n type = string\n}\n\nvariable \"mongodb_connection_uri\" {\n description = \"MongoDB connection URI used to connect to your MongoDB deployment\"\n type = string\n}\n```\n\nThe ``var-values.tfvars`` file holds the actual values and is passed to Terraform using the ``-var-file=`` flag.\n\n``var-values.tfvars`` file:\n\n```toml\naiven_api_token = \"\"\nproject_name = \"\"\nservice_name_prefix = \"\" \nmongodb_connection_uri = \"\"\n```\n\n## Services.tf file\n\n```toml\nresource \"aiven_kafka\" \"demo-kafka\" {\n project = var.project_name\n cloud_name = \"google-northamerica-northeast1\"\n plan = \"business-4\"\n service_name = join(\"-\", [var.service_name_prefix, \"kafka\"])\n maintenance_window_dow = \"sunday\"\n maintenance_window_time = \"10:00:00\"\n kafka_user_config {\n kafka_connect = true\n kafka_rest = true\n kafka_version = \"3.4\"\n kafka {\n auto_create_topics_enable = true\n }\n }\n}\n\nresource \"aiven_kafka_connector\" \"mongodb-source-connector\" {\n project = var.project_name\n service_name = aiven_kafka.demo-kafka.service_name\n connector_name = \"mongodb-source-connector\"\n config = {\n \"name\" = \"mongodb-source-connector\"\n \"connector.class\" = \"com.mongodb.kafka.connect.MongoSourceConnector\"\n \"connection.uri\" = var.mongodb_connection_uri\n \"database\" = \"sample_airbnb\"\n \"collection\" = \"listingsAndReviews\"\n \"copy.existing\" = \"true\"\n \"poll.await.time.ms\" = \"1000\"\n \"output.format.value\" = \"json\"\n \"output.format.key\" = \"json\"\n \"publish.full.document.only\" = \"true\"\n }\n}\n```\n\n## Execute the files \n\nThe ``init`` command performs several different initialization steps in order to prepare the current working directory for use with Terraform. In our case, this command automatically finds, downloads, and installs the necessary Aiven Terraform provider plugins.\n \n```shell\nterraform init\n```\n\nThe ``plan`` command creates an execution plan and shows you the resources that will be created (or modified) for you. This command does not actually create any resource; this is more like a preview.\n\n```bash\nterraform plan -var-file=var-values.tfvars\n```\n\nIf you're satisfied with the output of ``terraform plan``, go ahead and run the ``terraform apply`` command which actually does the task or creating (or modifying) your infrastructure resources. \n\n```bash\nterraform apply -var-file=var-values.tfvars\n```\n\n- Since you have ``kafka_connect`` set to ``true`` under the ``kafka_user_config``, you don't need a standalone Aiven for Apache Kafka Connect service.\n- The ``auto_create_topics_enable`` flag is enabled, therefore the connector is able to create the topic on the Apache Kafka cluster by pushing the first message, without having to create the topic first.\n- The automatically created topic name will be the concatenation of ``database`` and ``collection`` parameters - ``sample_airbnb.listingsAndReviews`` in this example.\n- ``poll.await.time.ms`` can be configured to set the amount of wait time before the MongoDB source connector pulls the new changes from a collection.\n- ``publish.full.document.only``, when set to ``true``, only publishes the actual document rather than the full change stream document. The default value of the parameter is ``false``.\n\n## More resources\n\nKeep in mind that some parameters and configurations will vary for your case. A reference to some of the advanced Apache Kafka configurations and other related resources:\n\n- [List of advanced Apache Kafka configurations](https://aiven.io/docs/products/kafka/kafka-connect/reference/advanced-params.html)\n- [Create a MongoDB source connector](https://aiven.io/docs/products/kafka/kafka-connect/howto/mongodb-poll-source-connector.html)\n- [List of available Apache Kafka® Connect connectors](https://aiven.io/docs/products/kafka/kafka-connect/concepts/list-of-connector-plugins.html)",{"_30":1413,"_1414":49,"_1415":1505},"2023-08-29T07:57:09.857Z",{"_43":1169,"_30":45},[1508],{"_38":1509,"_43":1422,"_30":1423},"5f3b772c1cd4",{"_30":1140,"_1143":1511},{"_30":814,"_817":1512},{"_43":1513,"_30":45},"image-f2771b9954ef0aef56fc48a9a90d09971d6cb94d-1567x844-png",{"_30":51,"_53":1515},"/developer/apache-kafka-with-mongodb","Use MongoDB as a source for Apache Kafka® topics and deploy the integration with both using Terraform. A part of Aiven's Terraform Cookbook.",[1518,1520],{"_38":1519,"_43":1290,"_30":1423},"300dea9cce88",{"_38":1521,"_43":1440,"_30":1423},"72efecccdfa2","Apache Kafka® with MongoDB source connector",{"_16":12,"_17":1524,"_19":1525,"_21":1526,"_30":1407,"_32":1527,"_1409":1528,"_1411":1529,"_1417":1531,"_1135":1532,"_1424":1482,"_1138":1535,"_51":1539,"_1433":1541,"_1151":1542,"_55":1550},"2023-08-21T09:43:44Z","7b31ec64-0438-419f-90a1-9e0835e2436a","VeNKqGNmt13PzvGhqsShiC","2024-01-29T15:45:55Z","From disaster recovery to isolating data for compliance reasons, businesses need to replicate data across their Apache Kafka® clusters, and Apache Kafka® MirrorMaker 2 is a perfect tool \nto do so. \n\nA single MirrorMaker 2 cluster can run multiple replication flows, and it has a mechanism for preventing replication cycles. This example sets up: \n\n* two [Aiven for Apache Kafka®](https://aiven.io/kafka) clusters (a source and a target)\n* an Aiven for Apache Kafka MirrorMaker 2 service\n* two service integrations between the Apache Kafka cluster and the MirrorMaker 2\n* a replication flow to move all the topics from the source cluster to the target cluster \n\nThe [Aiven Terraform Provider](https://registry.terraform.io/providers/aiven/aiven/latest/docs) is used to create all the required resources in a declarative style. The following image shows a unidirectional flow with the Apache Kafka MirrorMaker 2 replicating all the topics from ``DC1`` (source Kafka cluster) to ``DC2`` (target Kafka cluster):\n\n:::mermaid\n flowchart LR;\n\n subgraph DC2\n DC1.TopicA\n DC1.TopicB\n DC1.TopicC\n end;\n\n subgraph MM\n replication-flow-->DC1.TopicA\n replication-flow-->DC1.TopicB\n replication-flow-->DC1.TopicC\n end;\n\n subgraph DC1\n TopicA-->replication-flow\n TopicB-->replication-flow\n TopicC-->replication-flow\n end;\n:::\n\n## Describe the setup\n\nHere is the sample Terraform file that will spin up two Apache Kafka services, an Apache Kafka MirrorMaker 2 service and the MirrorMaker 2 service will be configured with two cluster alias pointed to the source and target Apache Kafka clusters. \n\nThe service integrations ``source-kafka-to-mm`` and ``mm-to-target-kafka`` connect the Kafka clusters to the MirrorMaker 2 instance. The replication flow ``mm-replication-flow`` creates a unidirectional flow to populate the remote topics based on source \ntopics. \n\nThe ``\".*\"`` wildcard in the MirrorMaker 2 configuration means that all the topics from the source cluster will be replicated to the target cluster. However, since the flow is unidirectional, the ``topic-b`` will only be present in the target cluster (where it was originally created) and not the source cluster.\n\nBe sure to check out the [getting started guide](https://aiven.io/docs/tools/terraform/get-started) to learn about the common files required to execute the following recipe. For example, you'll need to declare the variables for ``project_name`` and ``api_token``.\n\n## Common files\n\nNavigate to a new folder and add the following files.\n\nAdd the following to a new ``provider.tf`` file:\n\n```toml\nterraform {\n required_providers {\n aiven = {\n source = \"aiven/aiven\"\n version = \">=4.0.0, < 5.0.0\"\n }\n }\n}\n\nprovider \"aiven\" {\n api_token = var.aiven_api_token\n}\n```\n \nYou can also set the environment variable ``TF_VAR_aiven_api_token`` for the ``api_token`` property. With this, you don't need to pass the ``-var-file`` flag when executing Terraform commands.\n \nTo avoid including sensitive information in source control, the variables are defined here in the ``variables.tf`` file. You can then use a ``*.tfvars`` file with the actual values so that Terraform receives the values during runtime, and exclude it.\n\nThe ``variables.tf`` file defines the API token, the project name to use, and the prefix for the service name:\n\n```toml\nvariable \"aiven_api_token\" {\n description = \"Aiven console API token\"\n type = string\n}\n\nvariable \"project_name\" {\n description = \"Aiven console project name\"\n type = string\n}\n```\n\nThe ``var-values.tfvars`` file holds the actual values and is passed to Terraform using the ``-var-file=`` flag.\n\n``var-values.tfvars`` file:\n\n```toml\naiven_api_token = \"\"\nproject_name = \"\"\n```\n\n## Services.tf file\n\n``services.tf`` file:\n\n```toml\nresource \"aiven_kafka_mirrormaker\" \"mm\" {\n project = var.project_name\n cloud_name = \"google-europe-west1\"\n plan = \"business-4\"\n service_name = \"mm\"\n\n kafka_mirrormaker_user_config {\n ip_filter_string = [\n \"0.0.0.0/0\"\n ]\n\n kafka_mirrormaker {\n refresh_groups_interval_seconds = 600\n refresh_topics_enabled = true\n refresh_topics_interval_seconds = 600\n }\n }\n}\n\nresource \"aiven_service_integration\" \"source-kafka-to-mm\" {\n project = var.project_name\n integration_type = \"kafka_mirrormaker\"\n source_service_name = aiven_kafka.source.service_name\n destination_service_name = aiven_kafka_mirrormaker.mm.service_name\n\n kafka_mirrormaker_user_config {\n cluster_alias = \"source\"\n }\n}\n\nresource \"aiven_service_integration\" \"mm-to-target-kafka\" {\n project = var.project_name\n integration_type = \"kafka_mirrormaker\"\n source_service_name = aiven_kafka.target.service_name\n destination_service_name = aiven_kafka_mirrormaker.mm.service_name\n\n kafka_mirrormaker_user_config {\n cluster_alias = \"target\"\n }\n}\n\nresource \"aiven_mirrormaker_replication_flow\" \"mm-replication-flow\" {\n project = var.project_name\n service_name = aiven_kafka_mirrormaker.mm.service_name\n source_cluster = aiven_service_integration.source-kafka-to-mm.kafka_mirrormaker_user_config[0].cluster_alias\n target_cluster = aiven_service_integration.mm-to-target-kafka.kafka_mirrormaker_user_config[0].cluster_alias\n enable = true\n\n topics = [\n \".*\",\n ]\n\n topics_blacklist = [\n \".*[\\\\-\\\\.]internal\",\n \".*\\\\.replica\",\n \"__.*\"\n ]\n}\n\nresource \"aiven_kafka\" \"source\" {\n project = var.project_name\n cloud_name = \"google-europe-west1\"\n plan = \"business-4\"\n service_name = \"source\"\n maintenance_window_dow = \"monday\"\n maintenance_window_time = \"10:00:00\"\n\n kafka_user_config {\n kafka_version = \"3.2\"\n kafka {\n group_max_session_timeout_ms = 70000\n log_retention_bytes = 1000000000\n }\n }\n}\n\nresource \"aiven_kafka_topic\" \"source\" {\n project = var.project_name\n service_name = aiven_kafka.source.service_name\n topic_name = \"topic-a\"\n partitions = 3\n replication = 2\n}\n\nresource \"aiven_kafka\" \"target\" {\n project = var.project_name\n cloud_name = \"google-europe-west1\"\n plan = \"business-4\"\n service_name = \"target\"\n maintenance_window_dow = \"monday\"\n maintenance_window_time = \"10:00:00\"\n\n kafka_user_config {\n kafka_version = \"3.4\"\n kafka {\n group_max_session_timeout_ms = 70000\n log_retention_bytes = 1000000000\n }\n }\n}\n\nresource \"aiven_kafka_topic\" \"target\" {\n project = var.project_name\n service_name = aiven_kafka.target.service_name\n topic_name = \"topic-b\"\n partitions = 3\n replication = 2\n}\n```\n \n## Execute the files\n\nThe ``init`` command performs several different initialization steps in order to prepare the current working directory for use with Terraform. In our case, this command automatically finds, downloads, and installs the necessary Aiven Terraform provider plugins.\n\n```shell\nterraform init\n```\n\nThe ``plan`` command creates an execution plan and shows you the resources that will be created (or modified) for you. This command does not actually create any resource; this is more like a preview.\n\n```bash\nterraform plan -var-file=var-values.tfvars\n```\n\nIf you're satisfied with the output of ``terraform plan``, go ahead and run the ``terraform apply`` command which actually does the task or creating (or modifying) your infrastructure resources. \n\n```bash\nterraform apply -var-file=var-values.tfvars\n```\n\nFor Apache Kafka MirrorMaker 2 and Apache Kafka service integration, ``ip_filter_string`` is a specific configuration that whitelists certain ranges of IP addresses. This example of ``0.0.0.0/0`` denotes that all IP addresses are allowed.\n\nIn the target Apache Kafka cluster you will find: \n \n * the topic named ``topic-b`` created via the resource ``\"aiven_kafka_topic\" \"target\"``\n * some internal MirrorMaker 2 topics starting with prefix ``mm2``\n * a heartbeat topic for the ``source`` Kafka cluster named ``source.heartbeats``\n * the replicated topic ``topic-a`` prefixed with the source Kafka cluster alias ``source``\n\n## More resources\n\nKeep in mind that some parameters and configurations will vary for your case. A reference to some of the advanced Apache Kafka configurations and other related resources:\n\n- [Configuration options for Aiven for Apache Kafka](https://aiven.io/docs/products/kafka/reference/advanced-params.html)\n- [5 reasons why you should be using MirrorMaker 2.0 for data replication](https://aiven.io/blog/5-reasons-why-you-should-be-using-mirrormaker-2)\n- [Tutorial: Migrate an Apache Kafka® instance using Apache Kafka® MirrorMaker 2](https://aiven.io/developer/kafka-migration-mirrormaker2)\n- [Database migration with Apache Kafka® and Apache Kafka® Connect](https://aiven.io/developer/db-technology-migration-with-apache-kafka-and-kafka-connect)\n- [(Re)Create Apache Kafka® topics with Terraform](https://aiven.io/developer/terraform-for-repeatable-topics)\n- [Import your existing data platform using Terraform](https://aiven.io/developer/import-your-existing-data-platform-using-terraform)",{"_30":1413,"_1414":49,"_1415":1530},"2023-08-28T09:38:10.511Z",{"_43":1169,"_30":45},[1533],{"_38":1534,"_43":1422,"_30":1423},"ffc7eee82afb",{"_30":1140,"_1143":1536},{"_30":814,"_817":1537},{"_43":1538,"_30":45},"image-8e34a1ac341893fd1174924a231a083c8f01f76a-1567x844-png",{"_30":51,"_53":1540},"/developer/kafka-mirrormaker-crosscluster","Replicate data from one Apache Kafka® cluster to another using MirrorMaker 2. Useful when migrating Apache Kafka® services. A part of Aiven's Terraform Cookbook.",[1543,1545,1547],{"_38":1544,"_43":1290,"_30":1423},"18e78999da44",{"_38":1546,"_43":1440,"_30":1423},"be501531bb37",{"_38":1548,"_43":1549,"_30":1423},"5ac3dc41aec2","d4e858d3-c34f-479f-b156-3ad83e807091","Cross-cluster replication with Apache Kafka® MirrorMaker 2",{"_16":12,"_17":1552,"_19":1553,"_21":1554,"_30":1407,"_32":1555,"_1409":1556,"_1411":1557,"_1417":1559,"_1135":1560,"_1424":1482,"_1138":1563,"_51":1567,"_1433":1569,"_1151":1570,"_55":1575},"2023-08-21T11:28:13Z","b10e91b8-963a-415d-ae26-04a86831ab20","VeNKqGNmt13PzvGhqsSu3q","2024-01-29T15:48:45Z","\nThis example shows how to setup [Karapace](https://github.com/aiven/karapace) - an open source HTTP API interface and schema registry, with [Aiven for Apache Kafka®](https://aiven.io/kafka) using the [Aiven Terraform Provider](https://registry.terraform.io/providers/aiven/aiven/latest/docs).\n\nYou'll also enable the auto creation of Apache Kafka topics which will allow you to send message to topics that didn't exist already on the Apache Kafka cluster. In order to work directly with Kafka by producing and consuming messages over HTTP, the REST API feature will be enabled.\nTo learn more, check out [Create Apache Kafka® topics automatically](https://aiven.io/docs/products/kafka/howto/create-topics-automatically.html) page. \n\n:::mermaid\n\n flowchart LR\n producer[Producer]\n consumer[Consumer]\n SchemaRegistry{{Schema Registry:
Karapace}}\n producer --> Kafka --> consumer\n producer --> SchemaRegistry --> consumer\n\n:::\n\n## Describe the setup\n\nBe sure to check out the [getting started guide](https://aiven.io/docs/tools/terraform/get-started) to learn about the common files required to execute the following recipe. For example, you'll need to declare the variables for ``project_name`` and ``api_token``.\n\n## Common files \n\nNavigate to a new folder and add the following files.\n\nAdd the following to a new ``provider.tf`` file:\n\n```toml\nterraform {\n required_providers {\n aiven = {\n source = \"aiven/aiven\"\n version = \">=4.0.0, < 5.0.0\"\n }\n }\n}\n\nprovider \"aiven\" {\n api_token = var.aiven_api_token\n}\n```\n \nYou can also set the environment variable ``TF_VAR_aiven_api_token`` for the ``api_token`` property. With this, you don't need to pass the ``-var-file`` flag when executing Terraform commands.\n \nTo avoid including sensitive information in source control, the variables are defined here in the ``variables.tf`` file. You can then use a ``*.tfvars`` file with the actual values so that Terraform receives the values during runtime, and exclude it.\n\nThe ``variables.tf`` file defines the API token, the project name to use, and the prefix for the service name:\n\n```toml\nvariable \"aiven_api_token\" {\n description = \"Aiven console API token\"\n type = string\n}\n\nvariable \"project_name\" {\n description = \"Aiven console project name\"\n type = string\n}\n```\n \nThe ``var-values.tfvars`` file holds the actual values and is passed to Terraform using the ``-var-file=`` flag.\n\n``var-values.tfvars`` file:\n\n\n```toml\naiven_api_token = \"\"\nproject_name = \"\"\n```\n\n## Services.tf file\n\nHere is the sample Terraform file to stand-up a single Apache Kafka server and configure Karapace, Kafka REST, and auto creation of topics.\n\n``services.tf`` file:\n\n```toml\nresource \"aiven_kafka\" \"demo-kafka\" {\n project = var.project_name\n cloud_name = \"google-northamerica-northeast1\"\n plan = \"business-4\"\n service_name = \"demo-kafka\"\n maintenance_window_dow = \"monday\"\n maintenance_window_time = \"10:00:00\"\n kafka_user_config {\n kafka_version = \"3.4\"\n // Enables Karapace Schema Registry and REST\n schema_registry = true\n kafka_rest = true\n kafka {\n auto_create_topics_enable = true\n }\n }\n}\n\nresource \"aiven_kafka_topic\" \"source\" {\n project = var.project_name\n service_name = aiven_kafka.demo-kafka.service_name\n topic_name = \"topic-a\"\n partitions = 3\n replication = 2\n}\n```\n \n## Execute the files\n\nThe ``init`` command performs several different initialization steps in order to prepare the current working directory for use with Terraform. In our case, this command automatically finds, downloads, and installs the necessary Aiven Terraform provider plugins.\n \n```toml\nterraform init\n```\n\nThe ``plan`` command creates an execution plan and shows you the resources that will be created (or modified) for you. This command does not actually create any resource; this is more like a preview.\n\n\n```toml\nterraform plan -var-file=var-values.tfvars\n```\n\nIf you're satisfied with the output of ``terraform plan``, go ahead and run the ``terraform apply`` command which actually does the task or creating (or modifying) your infrastructure resources.\n \n\n```bash\nterraform apply -var-file=var-values.tfvars\n```\n \nLet's test that each of these configurations are setup by Terraform. Once the Aiven for Apache Kafka service is running, from the *Overview* tab, ensure that *Apache Kafka REST API (Karapace)* and *Schema Registry (Karapace)* are toggled on.\n\nFor documentation on how to use Karapace, refer to the [Karapace GitHub repository](https://github.com/aiven/karapace). \nWithout the REST API option enabled, you won't be able to view the messages in the topics from the Aiven web console. If you navigate to the *Topics* tab on Aiven console and are able to browse the messages for a particular topic (the *Messages* button is enabled), that confirms that the REST API setting has been enabled. \n\nFinally, you can send messages to a non-existing topic (for example, ``topic-b``) on your Apache Kafka cluster and the message will be delivered thanks to the ``auto_create_topics_enable`` parameter being set to ``true``.\nBy default, in Aiven for Apache Kafka this features is turned off as safeguard against accidental topic creation. Either remove this parameter from the Terraform code or set ``auto_create_topics_enable`` parameter to ``false`` and run the ``terraform apply`` again. \nThis time, you won't be able to send messages to a non-existing topic.\n\n## More resources\n\nTo find more information on Karapace and Apache Kafka:\n\n- [The Karapace project](https://www.karapace.io/)\n- [What is Apache Kafka](https://aiven.io/blog/what-is-apache-kafka)\n- [What is Karapace](https://aiven.io/blog/what-is-karapace)\n- [Karapace strengthens schema management](https://aiven.io/blog/karapace-strengthens-schema-management)\n",{"_30":1413,"_1414":49,"_1415":1558},"2023-08-28T11:25:34.007Z",{"_43":1169,"_30":45},[1561],{"_38":1562,"_43":1422,"_30":1423},"045824977162",{"_30":1140,"_1143":1564},{"_30":814,"_817":1565},{"_43":1566,"_30":45},"image-c2aadbe2aa16c873aadcdca0fbe78b86fc1b6daa-1567x844-png",{"_30":51,"_53":1568},"/developer/apache-kafka-karapace","Manage Apache Kafka® with the open source Karapace schema registry, and deploy both using Terraform. A part of Aiven's Terraform Cookbook.",[1571,1573],{"_38":1572,"_43":1290,"_30":1423},"6511c23ff609",{"_38":1574,"_43":1440,"_30":1423},"61d48cb1c83f","Apache Kafka® with Karapace Schema Registry",{"_16":12,"_17":1577,"_19":1578,"_21":1579,"_30":1407,"_32":1580,"_1409":1581,"_1411":1582,"_1417":1584,"_1135":1585,"_1424":1482,"_1138":1588,"_51":1592,"_1433":1594,"_1151":1595,"_55":1600},"2023-08-22T09:20:29Z","d4596bb2-1427-497d-b642-ea2c94095067","VeNKqGNmt13PzvGhqsSuTu","2024-01-29T15:48:51Z","This example deploys an [Aiven for Apache Kafka®](https://aiven.io/kafka) service with some custom configurations, as well as a Kafka topic, a Kafka user, and an access control list (ACL) to allow fine-grained permissions about which topic that user can access, using the [Aiven Terraform Provider](https://registry.terraform.io/providers/aiven/aiven/latest/docs)\n\nBefore looking at the Terraform script, let's visualize the resources:\n\n:::mermaid\n\n flowchart LR\n subgraph Aiven-for-Apache-Kafka\n id4[[Aiven Kafka User]]\n id5[[Aiven Kafka User ACL]]\n end\n Producer --> Aiven-for-Apache-Kafka --> Consumer\n\n:::\n\nBe sure to check out the [getting started guide](https://aiven.io/docs/tools/terraform/get-started) to learn about the common files required to execute the following recipe. For example, you'll need to declare the variables for ``project_name``, ``api_token``, and ``kafka_user_name``.\n\n## Common files\n\n\nNavigate to a new folder and add the following files.\n\nAdd the following to a new ``provider.tf`` file:\n\n```toml\nterraform {\n required_providers {\n aiven = {\n source = \"aiven/aiven\"\n version = \">=4.0.0, < 5.0.0\"\n }\n }\n}\n\nprovider \"aiven\" {\n api_token = var.aiven_api_token\n}\n```\n\nYou can also set the environment variable ``TF_VAR_aiven_api_token`` for the ``api_token`` property. With this, you don't need to pass the ``-var-file`` flag when executing Terraform commands.\n\nTo avoid including sensitive information in source control, the variables are defined here in the ``variables.tf`` file. You can then use a ``*.tfvars`` file with the actual values so that Terraform receives the values during runtime, and exclude it.\n\nThe ``variables.tf`` file defines the API token, the project name to use, and the prefix for the service name:\n\n```toml\nvariable \"aiven_api_token\" {\n description = \"Aiven console API token\"\n type = string\n}\n\nvariable \"project_name\" {\n description = \"Aiven console project name\"\n type = string\n}\n\nvariable \"kafka_user_name\" {\n description = \"Username for Aiven for Apache Kafka user\"\n type = string\n} \n```\n\nThe ``var-values.tfvars`` file holds the actual values and is passed to Terraform using the ``-var-file=`` flag.\n\n``var-values.tfvars`` file:\n\n```toml\naiven_api_token = \"\"\nproject_name = \"\"\nkafka_user_name = \"\"\n```\n\n## Services.tf file \n\nHere is the sample Terraform script to stand-up Aiven for Apache Kafka and related resources. The script also performs some custom configurations on these resources.\n\n``services.tf`` file:\n\n```toml\nresource \"aiven_kafka\" \"demo-kafka\" {\n project = var.project_name\n cloud_name = \"google-europe-west1\"\n plan = \"business-4\"\n service_name = \"demo-kafka\"\n maintenance_window_dow = \"sunday\"\n maintenance_window_time = \"01:00:00\"\n default_acl = false\n\n kafka_user_config {\n kafka_rest = true\n kafka_connect = false\n schema_registry = true\n kafka_version = \"3.4\"\n\n kafka {\n auto_create_topics_enable = true\n num_partitions = 3\n default_replication_factor = 2\n min_insync_replicas = 2\n }\n\n kafka_authentication_methods {\n certificate = true\n }\n\n public_access {\n kafka_rest = true\n }\n }\n}\n\nresource \"aiven_kafka_topic\" \"demo-kafka-topic\" {\n project = var.project_name\n service_name = aiven_kafka.demo-kafka.service_name\n topic_name = \"demo-kafka-topic\"\n partitions = 5\n replication = 3\n}\n\nresource \"aiven_kafka_user\" \"demo-kafka-user\" {\n project = var.project_name\n service_name = aiven_kafka.demo-kafka.service_name\n username = var.kafka_user_name\n}\n\nresource \"aiven_kafka_acl\" \"demo-kafka-user-acl\" {\n project = var.project_name\n service_name = aiven_kafka.demo-kafka.service_name\n username = var.kafka_user_name\n permission = \"read\"\n topic = aiven_kafka_topic.demo-kafka-topic.topic_name\n}\n```\n \n## Execute the files \n\nThe ``init`` command performs several different initialization steps in order to prepare the current working directory for use with Terraform. In our case, this command automatically finds, downloads, and installs the necessary Aiven Terraform provider plugins.\n \n```shell\nterraform init\n```\n\nThe ``plan`` command creates an execution plan and shows you the resources that will be created (or modified) for you. This command does not actually create any resource; this is more like a preview.\n\n```bash\nterraform plan -var-file=var-values.tfvars\n```\n\nIf you're satisfied with the output of ``terraform plan``, go ahead and run the ``terraform apply`` command which actually does the task or creating (or modifying) your infrastructure resources. \n\n```bash\nterraform apply -var-file=var-values.tfvars\n```\n\nLet's go over a few of these configurations and understand their functions:\n\n``aiven_kafka`` resource configurations:\n\n- ``default_acl`` parameter, when set to **true**, creates default wildcard Kafka ACL. This example sets this parameter to **false** and prevents the default wildcard ACL for resources.\n\n- For ``kafka_user_config``, ``schema_registry`` is set to **true**, which enables the `Karapace Schema Registry `_ and ``kafka_rest`` allows you to view the messages in the topics from the Aiven web console when set to **true**.\n\n- ``auto_create_topics_enable`` under ``kafka`` nested configurations enables the auto creation of topics when set to **true**. This means that a topic doesn't need to exist before sending a message.\n\n- ``num_partitions`` will set the number of partitions for the automatically created topics.\n\n- By default, the replication factor is 2. This example sets ``default_replication_factor`` to 2 and thus requires a minimum of two brokers. For production environments, a replication factor of 3 is recommended. \n\n- ``min_insync_replicas`` indicates that at least 2 replicas (brokers) should respond back if all replicas(brokers) are not functioning properly. When all replicas are functioning properly, this setting has no effect. \n\n- The ``certificate`` parameter under the ``kafka_authentication_methods`` nested configurations, when set to **true**, enables certificate/SSL authentication.\n\n``aiven_kafka_topic`` resource configurations:\n\n- ``partitions`` denotes the number of partitions to create in the topic, and ``replication`` sets the replication factor for the topic.\n\n``aiven_kafka_user`` resource configurations:\n\n- We are passing a preset username using ``var.kafka_user_name``\n\n``aiven_kafka_acl`` resource configurations:\n\n- This ACL allows **read** access to the ``demo-kafka-topic`` topic for the ``var.kafka_user_name`` user. \n\n::alert[By default, Aiven adds an ``avnadmin`` account to every new service and adds `admin` permission for all topics to that user. When you create your own ACLs to restrict access, you probably want to remove this ACL entry.]{title=\"Warning\"}\n\n::alert[When using the Aiven Terraform Provider, you can add the ``default_acl`` key to your ``resource`` and set it to ``false`` if you do not want to create the admin user with wildcard permissions.]{title=\"Note\"}\n\nMore resources\n--------------\n\nKeep in mind that some parameters and configurations will vary for your case. Some related resources are provided below:\n\n- [Configuration options for Aiven for Apache Kafka](https://aiven.io/docs/products/kafka/reference/advanced-params.html)\n- [Aiven for Apache Kafka access control lists permission mapping](https://aiven.io/docs/products/kafka/concepts/acl.html)\n- [How to Manage Aiven for Apache Kafka Parameters](https://www.youtube.com/watch?v=pXQZWI0ddLg)\n- [Set up your first Aiven Terraform project](https://aiven.io/docs/tools/terraform/get-started.html)",{"_30":1413,"_1414":49,"_1415":1583},"2023-08-29T09:20:07.943Z",{"_43":1169,"_30":45},[1586],{"_38":1587,"_43":1422,"_30":1423},"de5df780f165",{"_30":1140,"_1143":1589},{"_30":814,"_817":1590},{"_43":1591,"_30":45},"image-dfb7670755fa79dd22f530c2a23ce8841d166754-1567x844-png",{"_30":51,"_53":1593},"/developer/apache-kafka-with-custom-configurations","Deploy an Apache Kafka® node to Aiven for Apache Kafka® with custom configurations defined in Terraform. A part of Aiven's Terraform Cookbook.",[1596,1598],{"_38":1597,"_43":1290,"_30":1423},"478a10066927",{"_38":1599,"_43":1440,"_30":1423},"56f4fc2dd393","Apache Kafka® with custom configurations",{"_16":12,"_17":1602,"_19":1603,"_21":1604,"_23":1605,"_30":1407,"_32":1608,"_1409":1609,"_1411":1610,"_1417":1612,"_1135":1613,"_1424":1616,"_1138":1617,"_51":1622,"_1433":1624,"_1151":1625,"_55":1632},"2023-08-21T07:46:05Z","2038f4c7-4494-4d3a-964e-7cf95e608128","g8eHHsLV7ci6mnlkKzmvq6",{"_25":1606},{"_27":1603,"_28":1607},"VeNKqGNmt13PzvGhqsSdc4","2025-09-30T03:40:33Z","This example shows how to set up an [Aiven for Apache Kafka®](https://aiven.io/kafka) service with an [Aiven for Apache Flink®](https://aiven.io/flink) integration using the [Aiven Terraform Provider](https://registry.terraform.io/providers/aiven/aiven/latest/docs). An Apache Kafka source topic is used as a data source, and Apache Flink processes the data to do filtering or transformation, and finally write the transformed output to a sink topic.\nPlease get yourself familiar with [Aiven for Apache Flink concepts](https://aiven.io/docs/products/flink/concepts) before you start cooking.\n\nBefore looking at the Terraform script, let's visually realize how the services will be connected:\n\n:::mermaid\n\n flowchart LR\n subgraph Aiven for Apache Kafka\n SourceTopic\n end\n subgraph Aiven for Apache Flink Application\n SourceTopic-->|FlinkTableMapping|FlinkSourceTable\n FlinkSourceTable --> SQLtransformation\n SQLtransformation --> FlinkTargetTable\n end\n subgraph Aiven for Apache Kafka\n FlinkTargetTable-->|FlinkTableMapping|SinkTopic\n end\n:::\n\nImagine that you are collecting CPU usage for hundreds of machines in your data centre and these metrics are populated in an Apache Kafka topic called ``cpu_measurements``. But you're interested in learning about those machines with CPU usages higher than 85% and write the filtered messages into a topic called ``cpu_high_usage``.\n\nIf you relate the above diagram to this example, both source and target Apache Kafka topics are part of the same Apache Kafka cluster. To do the processing on the data, you'll be using an Aiven for Apache Flink Application which is an abstraction layer on top of Apache Flink SQL that includes all the elements related to a Flink job to help build your data processing pipeline. \n\nThe following Terraform script stands up both Apache Kafka and Apache Flink services, creates the service integration, source and target Apache Kafka topics, an Aiven for Apache Flink application, and the Aiven for Apache Flink application version. By design, you cannot manage the deployment resource using Aiven Terraform Provider. \nIn order to do so, you'll need to use Aiven console or Aiven CLI.\n\n## Common files\n\nNavigate to a new folder and add the following files.\n\nAdd the following to a new ``provider.tf`` file:\n\n```toml\nterraform {\n required_providers {\n aiven = {\n source = \"aiven/aiven\"\n version = \">=4.0.0, < 5.0.0\"\n }\n }\n}\n\nprovider \"aiven\" {\n api_token = var.aiven_api_token\n}\n```\n\nYou can also set the environment variable ``TF_VAR_aiven_api_token`` for the ``api_token`` property and ``TF_VAR_project_name`` for the ``project_name`` property. With this, you don't need to pass the ``-var-file`` flag when executing Terraform commands.\n \nTo avoid including sensitive information in source control, the variables are defined here in the ``variables.tf`` file. You can then use a ``*.tfvars`` file with the actual values so that Terraform receives the values during runtime, and exclude it.\n\nThe ``variables.tf`` file defines the API token, the project name to use, and the prefix for the service name:\n\n```toml\nvariable \"aiven_api_token\" {\n description = \"Aiven console API token\"\n type = string\n}\n\nvariable \"project_name\" {\n description = \"Aiven console project name\"\n type = string\n}\n```\n \nThe ``var-values.tfvars`` file holds the actual values and is passed to Terraform using the ``-var-file=`` flag.\n\n``var-values.tfvars`` file:\n\n```toml\naiven_api_token = \"\"\nproject_name = \"\"\n```\n\n\n## Services.tf file\n\n```toml\n# Flink service\n\nresource \"aiven_flink\" \"demo-flink\" {\n project = var.project_name\n cloud_name = \"google-northamerica-northeast1\"\n plan = \"business-8\"\n service_name = \"demo-flink\"\n}\n\n# Kafka service\n\nresource \"aiven_kafka\" \"demo-kafka\" {\n project = var.project_name\n cloud_name = \"google-northamerica-northeast1\"\n plan = \"business-8\"\n service_name = \"demo-kafka\"\n}\n\n# Flink-Kafka integration\n\nresource \"aiven_service_integration\" \"flink_to_kafka\" {\n project = var.project_name\n integration_type = \"flink\"\n destination_service_name = aiven_flink.demo-flink.service_name\n source_service_name = aiven_kafka.demo-kafka.service_name\n}\n\n# Flink application\n\nresource \"aiven_flink_application\" \"demo-flink-app\" {\n project = var.project_name\n service_name = aiven_flink.demo-flink.service_name\n name = \"demo-flink-app\"\n}\n\n# Flink application version\n\nresource \"aiven_flink_application_version\" \"demo-flink-app-version\" {\n project = var.project_name\n service_name = aiven_flink.demo-flink.service_name\n application_id = aiven_flink_application.demo-flink-app.application_id\n statement = < 85\nEOT\nsinks {\n create_table = < OpenSearch Sink}}\n opensearch[OpenSearch]\n kafka --> sikafka --> kconn --> kcopensearch --> opensearch\n\n:::\n\nIn the above diagram, *KafkaConnect* is the service that you create for connecting Kafka with external systems. The Kafka Connectors, *OpenSearch Sink Connector* for example, are ready-to-use components to send/receive data to common data sources/sinks. \n\n## Describe the setup\n\nBe sure to check out the [getting started guide](https://aiven.io/docs/tools/terraform/get-started) to learn about the common files required to execute the following recipe. For example, you'll need to declare the variables for ``project_name`` and ``api_token``.\n\n## Common files\n\nNavigate to a new folder and add the following files.\n\nAdd the following to a new ``provider.tf`` file:\n\n```toml\n terraform {\n required_providers {\n aiven = {\n source = \"aiven/aiven\"\n version = \">=4.0.0, < 5.0.0\"\n }\n }\n }\n\n provider \"aiven\" {\n api_token = var.aiven_api_token\n }\n```\n \nYou can also set the environment variable ``TF_VAR_aiven_api_token`` for the ``api_token`` property. With this, you don't need to pass the ``-var-file`` flag when executing Terraform commands.\n \n\nTo avoid including sensitive information in source control, the variables are defined here in the ``variables.tf`` file. You can then use a ``*.tfvars`` file with the actual values so that Terraform receives the values during runtime, and exclude it.\n\nThe ``variables.tf`` file defines the API token, the project name to use, and the prefix for the service name:\n\n```toml\nvariable \"aiven_api_token\" {\n description = \"Aiven console API token\"\n type = string\n}\n\nvariable \"project_name\" {\n description = \"Aiven console project name\"\n type = string\n}\n```\n \nThe ``var-values.tfvars`` file holds the actual values and is passed to Terraform using the ``-var-file=`` flag.\n\n``var-values.tfvars`` file:\n\n```toml\naiven_api_token = \"\"\nproject_name = \"\"\n```\n\n## Services.tf file\n\nHere is the sample Terraform file to stand-up and connect all the services. Keep in mind that some parameters and configurations will vary for your case. A reference to the Kafka and OpenSearch configurations are added at the end of this document.\n\n``services.tf`` file:\n\n```toml \n# Kafka service\nresource \"aiven_kafka\" \"application-logs\" {\n project = var.project_name\n cloud_name = \"google-northamerica-northeast1\"\n plan = \"business-4\"\n service_name = \"kafka-application-logs\"\n maintenance_window_dow = \"monday\"\n maintenance_window_time = \"10:00:00\"\n kafka_user_config {\n kafka_connect = true\n kafka_rest = true\n kafka_version = \"3.4\"\n kafka {\n group_max_session_timeout_ms = 70000\n log_retention_bytes = 1000000000\n }\n }\n}\n\n# Kafka topic\nresource \"aiven_kafka_topic\" \"topic-logs-app-1\" {\n project = var.project_name\n service_name = aiven_kafka.application-logs.service_name\n topic_name = \"logs-app-1\"\n partitions = 3\n replication = 2\n}\n\n# Kafka connect service\nresource \"aiven_kafka_connect\" \"logs-connector\" {\n project = var.project_name\n cloud_name = \"google-northamerica-northeast1\"\n plan = \"business-4\"\n service_name = \"kafka-connect-logs-connector\"\n maintenance_window_dow = \"monday\"\n maintenance_window_time = \"10:00:00\"\n kafka_connect_user_config {\n kafka_connect {\n consumer_isolation_level = \"read_committed\"\n }\n public_access {\n kafka_connect = true\n }\n }\n}\n\n# Kafka connect service integration\nresource \"aiven_service_integration\" \"kafka-to-logs-connector\" {\n project = var.project_name\n integration_type = \"kafka_connect\"\n source_service_name = aiven_kafka.application-logs.service_name\n destination_service_name = aiven_kafka_connect.logs-connector.service_name\n kafka_connect_user_config {\n kafka_connect {\n group_id = \"connect\"\n status_storage_topic = \"__connect_status\"\n offset_storage_topic = \"__connect_offsets\"\n }\n }\n}\n\n# Kafka connector\nresource \"aiven_kafka_connector\" \"kafka-os-con1\" {\n project = var.project_name\n service_name = aiven_kafka.application-logs.service_name\n connector_name = \"kafka-os-con1\"\n config = {\n \"topics\" = aiven_kafka_topic.topic-logs-app-1.topic_name\n \"connector.class\" = \"io.aiven.kafka.connect.opensearch.OpensearchSinkConnector\"\n \"type.name\" = \"os-connector\"\n \"name\" = \"kafka-os-con1\"\n \"connection.url\" = \"https://${aiven_opensearch.os-service1.service_host}:${aiven_opensearch.os-service1.service_port}\"\n \"connection.username\" = sensitive(aiven_opensearch.os-service1.service_username)\n \"connection.password\" = sensitive(aiven_opensearch.os-service1.service_password)\n \"key.converter\" = \"org.apache.kafka.connect.storage.StringConverter\"\n \"value.converter\" = \"org.apache.kafka.connect.json.JsonConverter\"\n \"tasks.max\" = 1\n \"schema.ignore\" = true\n \"value.converter.schemas.enable\" = false\n }\n}\n\n# Opensearch service\nresource \"aiven_opensearch\" \"os-service1\" {\n project = var.project_name\n cloud_name = \"google-northamerica-northeast1\"\n plan = \"business-4\"\n service_name = \"os-service1\"\n maintenance_window_dow = \"monday\"\n maintenance_window_time = \"10:00:00\"\n opensearch_user_config {\n opensearch_version = \"2\"\n }\n}\n```\n\n# Execute the files\n\nThe ``init`` command performs several different initialization steps in order to prepare the current working directory for use with Terraform. In our case, this command automatically finds, downloads, and installs the necessary Aiven Terraform provider plugins.\n \n```shell\nterraform init\n```\n\nThe ``plan`` command creates an execution plan and shows you the resources that will be created (or modified) for you. This command does not actually create any resource; this is more like a preview.\n\n```bash\nterraform plan -var-file=var-values.tfvars\n```\n\nIf you're satisfied with the output of ``terraform plan``, go ahead and run the ``terraform apply`` command which actually does the task or creating (or modifying) your infrastructure resources. \n\n```bash\nterraform apply -var-file=var-values.tfvars\n```\n \nThis file creates three Aiven services - a Kafka service, a Kafka Connect service, and an OpenSearch service. Two service integrations among these three services and a Kafka topic within the Kafka service will also be created from this Terraform file.\nTo validate, produce some messages on the Kafka topic and you should be seeing those appear on OpenSearch indices.\n\n## More resources\n\nYou might find these related resources useful too:\n\n- [Configuration options for Kafka](https://aiven.io/docs/products/kafka/reference/advanced-params.html)\n- [Configuration options for OpenSearch](https://aiven.io/docs/products/opensearch/reference/advanced-params.html)\n- [Aiven for Apache Kafka®](https://aiven.io/kafka)\n- [Aiven for OpenSearch®![](https://cdn.sanity.io/images/sczeoy4w/production/c7c4ac032a066462569b6b3631c1e0976a5c84a7-468x15.png?w=450)](https://aiven.io/opensearch)\n",{"_30":1413,"_1414":49,"_1415":1640},"2023-08-23T13:13:18.345Z",{"_43":1169,"_30":45},[1643],{"_38":1644,"_43":1422,"_30":1423},"0bf9fec103e9",{"_30":1140,"_1142":1646,"_1143":1647},"Learn how to connect Apache Kafka® to OpenSearch® with Terraform. Improve your workflows today and find out more in the Aiven Developer Center!",{"_30":814,"_815":1648,"_817":1649},"Connect Apache Kafka to OpenSearch with Terraform",{"_43":1650,"_30":45},"image-4888da7cef670e4a85349180a212407e40d45136-1567x844-png",{"_30":51,"_53":1652},"/developer/apache-kafka-to-opensearch-terraform","Use Terraform to connect Apache Kafka® to OpenSearch®. A part of Aiven's Terraform Cookbook series.",[1655,1657],{"_38":1656,"_43":1290,"_30":1423},"c6ba1c8f1422",{"_38":1658,"_43":1440,"_30":1423},"5fde2a9761af","Connect Apache Kafka® to OpenSearch® with Terraform","This cookbook provides copy-and-paste ready Terraform examples with various Aiven services and integrations. They are designed for people looking for a quick solution to a specific problem. Interested in seeing more cookbooks? Leave a comment for us on the forum!",{"_30":1413,"_1414":47,"_1415":1662},"2030-04-01T07:11:00.000Z",{"_9":1170,"_51":1664},{"_30":51,"_53":547},"2024-03-27T07:58:03.169Z",{"_30":51,"_53":10},"This is a collection of Terraform Cookbook for Apache Kafka",[1669,1671],{"_19":1290,"_51":1670,"_55":1193},{"_30":51,"_53":1192},{"_19":1440,"_51":1672,"_55":1202},{"_30":51,"_53":1201},"Terraform Cookbook (Apache Kafka)",[],"articles","collectionMenu",{"_19":1395,"_55":1673,"_1433":1667,"_51":10,"_1675":1678},[1679,1680,1681,1682,1683,1684,1685,1686,1687],{"_19":1402,"_55":1441,"_1433":1434,"_51":1432},{"_19":1444,"_55":1469,"_1433":1461,"_51":1460},{"_19":1472,"_55":1497,"_1433":1489,"_51":1488},{"_19":1500,"_55":1522,"_1433":1516,"_51":1515},{"_19":1525,"_55":1550,"_1433":1541,"_51":1540},{"_19":1553,"_55":1575,"_1433":1569,"_51":1568},{"_19":1578,"_55":1600,"_1433":1594,"_51":1593},{"_19":1603,"_55":1632,"_1433":1624,"_51":1623},{"_19":1635,"_55":1659,"_1433":1653,"_51":1652},"collectionPagination",{"_53":-5,"_1690":-5,"_1691":1679},"previous","next","collectionList",[1694],{"_19":1395,"_55":1673,"_1433":1667,"_51":10},"actionData","errors"]