{"id":10880,"date":"2024-12-11T09:40:24","date_gmt":"2024-12-11T09:40:24","guid":{"rendered":"https:\/\/www.aihello.com\/resources\/?p=10880"},"modified":"2026-09-23T02:13:11","modified_gmt":"2026-09-23T02:13:11","slug":"building-a-rag-pipeline-with-fastapi-haystack-and-chromadb-for-urls-in-python","status":"publish","type":"post","link":"https:\/\/www.aihello.com\/resources\/blog\/building-a-rag-pipeline-with-fastapi-haystack-and-chromadb-for-urls-in-python\/","title":{"rendered":"Building a RAG Pipeline with FastAPI, Haystack, and ChromaDB for URLs in Python"},"content":{"rendered":"<div class=\"wp-block-group ah-reading-guide ah-compact-toc\"><div class=\"wp-block-group__inner-container is-layout-flow wp-block-group-is-layout-flow\"><style>\n.ah-reading-guide{max-width:820px;margin:0 auto;color:#26364a;font-size:18px;line-height:1.8;counter-reset:ah-section}.ah-reading-guide *{box-sizing:border-box}.ah-reading-guide p{margin:0 0 22px;line-height:1.8}.ah-reading-guide strong{color:#142d48}.ah-reading-guide h2{position:relative;margin:56px 0 24px;padding-top:24px;border-top:1px solid #dce7ef;font-size:29px;line-height:1.35;letter-spacing:-.4px;color:#142d48}.ah-reading-guide h2.ah-numbered{padding-left:54px;counter-increment:ah-section}.ah-reading-guide h2.ah-numbered:before{content:counter(ah-section,decimal-leading-zero);position:absolute;left:0;top:26px;display:flex;align-items:center;justify-content:center;width:38px;height:38px;background:#eaf4fc;color:#216faf;border-radius:10px;font-size:15px;font-weight:700;letter-spacing:0}.ah-reading-guide h3{position:relative;margin:34px 0 16px;font-size:23px;line-height:1.4;color:#163f65}.ah-reading-guide h4{position:relative;margin:28px 0 14px;font-size:20px;line-height:1.45;color:#163f65}.ah-reading-guide .ah-lead{margin:0 0 28px;padding:24px 28px;border-left:4px solid #2479ba;border-radius:0 12px 12px 0;background:#eef6fc;font-size:20px;line-height:1.75}.ah-reading-guide .ah-callout,.ah-reading-guide .ah-formula{margin:28px 0;padding:24px 26px;border:1px solid #d7e5f1;border-left:4px solid #2479ba;border-radius:0 12px 12px 0;background:#f5f9fd}.ah-reading-guide .ah-formula{font-size:20px;font-weight:600;font-variant-numeric:tabular-nums;line-height:1.9;overflow-wrap:anywhere}.ah-reading-guide ul,.ah-reading-guide ol{margin:24px 0 30px;padding-left:28px}.ah-reading-guide li{margin:0 0 12px;padding-left:7px;line-height:1.75}.ah-reading-guide li::marker{color:#2479ba;font-weight:700}.ah-reading-guide ol.ah-steps{list-style:none;counter-reset:ah-step;padding:0;display:grid;gap:14px}.ah-reading-guide ol.ah-steps>li{position:relative;counter-increment:ah-step;margin:0;padding:22px 24px 22px 64px;border:1px solid #dbe7f1;border-radius:12px;background:#f6fafd}.ah-reading-guide ol.ah-steps>li:before{content:counter(ah-step);position:absolute;top:23px;left:20px;display:flex;align-items:center;justify-content:center;width:28px;height:28px;border-radius:50%;background:#2378b9;color:white;font-size:14px;font-weight:700}.ah-reading-guide ul.ah-cards{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:14px;padding:0;list-style:none}.ah-reading-guide ul.ah-cards>li{margin:0;padding:20px 22px;border:1px solid #dbe8f2;background:#f7fbfe;border-radius:12px;font-size:17px;line-height:1.7}.ah-reading-guide ul.ah-cards>li:last-child:nth-child(odd){grid-column:1\/-1}.ah-reading-guide .ah-cards>li>strong:first-child{display:block;margin-bottom:8px;color:#19598d}.ah-reading-guide .ah-step-panel{margin:22px 0;padding:24px 28px;border-left:4px solid #2479ba;border-radius:0 12px 12px 0;background:#f5f9fd}.ah-reading-guide .ah-step-panel h3{margin-top:0}.ah-reading-guide .ah-step-panel p:last-child{margin-bottom:0}.ah-reading-guide figure{max-width:100%;margin:30px 0}.ah-reading-guide figure.wp-block-image{padding:7px;border:1px solid #e0e8ef;border-radius:12px;background:white;box-shadow:0 5px 18px rgba(20,45,72,.06)}.ah-reading-guide img{max-width:100%;height:auto}.ah-reading-guide figure.wp-block-image img{display:block;border-radius:6px}.ah-reading-guide figcaption{font-size:14px;line-height:1.5;color:#587086;margin-top:10px}.ah-reading-guide figure.wp-block-table{overflow-x:auto;border:1px solid #d8e5ef;border-radius:10px}.ah-reading-guide table{width:100%;border-collapse:collapse;font-size:16px;line-height:1.6}.ah-reading-guide th,.ah-reading-guide td{padding:14px 16px;border:1px solid #d8e5ef;text-align:left;vertical-align:top}.ah-reading-guide th{background:#eaf4fc;color:#163f65;font-weight:700}.ah-reading-guide tbody tr:nth-child(even){background:#f6f9fc}.ah-reading-guide blockquote{margin:30px 0;padding:22px 26px;border-left:4px solid #2479ba;background:#f3f8fc;border-radius:0 10px 10px 0}.ah-reading-guide #ez-toc-container{width:100%;margin:32px 0 42px;padding:22px 24px;border:1px solid #cbdfee;border-top:4px solid #2479ba;border-radius:12px;background:#f7fbfe;box-shadow:none}.ah-reading-guide #ez-toc-container .ez-toc-title{margin:0;font-size:20px;font-weight:700;color:#163f65;line-height:1.5}.ah-reading-guide #ez-toc-container .ez-toc-title-container{margin-bottom:14px}.ah-reading-guide #ez-toc-container ul.ez-toc-list-level-1{display:block;column-count:2;column-gap:26px;padding:0;margin:0}.ah-reading-guide #ez-toc-container ul.ez-toc-list-level-1>li{break-inside:avoid;padding:11px 0;margin:0;border-bottom:1px solid #dce7ef;line-height:1.5;font-size:16px}.ah-reading-guide #ez-toc-container ul ul{margin:7px 0 0;padding-left:18px}.ah-reading-guide #ez-toc-container li li{padding:0;margin:5px 0;font-size:14px}.ah-reading-guide #ez-toc-container a{color:#19598d;text-decoration:none}.ah-reading-guide #ez-toc-container a:hover{text-decoration:underline}.ah-reading-guide #ez-toc-container a:focus-visible{outline:3px solid #2479ba;outline-offset:4px;border-radius:3px}.ah-reading-guide :is(h2,h3,h4)>.ez-toc-section{position:absolute;top:-100px;left:0}.ah-reading-guide :is(h2,h3,h4)>.ez-toc-section-end{position:absolute;top:0;left:0}.ah-reading-guide [id]{scroll-margin-top:100px}.ah-reading-guide .gspb_container{min-width:0}@media(max-width:700px){.ah-reading-guide{font-size:17px}.ah-reading-guide h2{font-size:25px;margin-top:44px}.ah-reading-guide h2.ah-numbered{padding-left:43px}.ah-reading-guide h2.ah-numbered:before{width:30px;height:30px;font-size:13px}.ah-reading-guide h3{font-size:21px}.ah-reading-guide ul.ah-cards{grid-template-columns:1fr}.ah-reading-guide .ah-lead,.ah-reading-guide .ah-callout,.ah-reading-guide .ah-formula,.ah-reading-guide .ah-step-panel{padding:20px}.ah-reading-guide .ah-lead,.ah-reading-guide .ah-formula{font-size:18px}.ah-reading-guide #ez-toc-container{padding:18px}.ah-reading-guide #ez-toc-container ul.ez-toc-list-level-1{column-count:1}}@media(prefers-reduced-motion:reduce){.ah-reading-guide [data-aos]{opacity:1!important;visibility:visible!important;transform:none!important;transition:none!important;animation:none!important}}\n\n.ah-reading-guide .ah-process-heading{padding-left:48px;min-height:36px}.ah-reading-guide .ah-process-heading:before{position:absolute;left:0;top:0;width:34px;height:34px;display:flex;align-items:center;justify-content:center;border-radius:9px;background:#eaf4fc;color:#216faf;font-size:15px;font-weight:700}.ah-reading-guide .ah-process-1:before{content:\"01\"}.ah-reading-guide .ah-process-2:before{content:\"02\"}.ah-reading-guide .ah-process-3:before{content:\"03\"}.ah-reading-guide .ah-process-4:before{content:\"04\"}.ah-reading-guide .ah-process-5:before{content:\"05\"}.ah-reading-guide .ah-process-6:before{content:\"06\"}.ah-reading-guide .ah-process-7:before{content:\"07\"}.ah-reading-guide .ah-process-8:before{content:\"08\"}.ah-reading-guide .ah-process-9:before{content:\"09\"}\n.ah-reading-guide .ah-definition{padding:20px 24px;margin:14px 0;border:1px solid #dbe7f1;border-radius:12px;background:#f8fbfe}.ah-reading-guide .ah-definition strong{color:#163f65}\n.ah-reading-guide .ah-long-form-list>li{margin-bottom:28px}.ah-reading-guide .ah-long-form-list>li>strong:first-child{display:block;font-size:22px;line-height:1.4;margin-bottom:16px}.ah-reading-guide .ah-long-form-list li img{display:block;margin:22px auto}.ah-reading-guide .ah-formula-comparison{font-variant-numeric:tabular-nums}\n.ah-reading-guide h5,.ah-reading-guide h6{font-size:18px;line-height:1.5;color:#163f65;margin:26px 0 14px;font-weight:700}\n.ah-reading-guide h2.ah-instruction-heading{font-size:21px;margin:24px 0 18px;padding-top:18px}.ah-reading-guide h2.ah-instruction-heading:before{top:18px}\n.ah-reading-guide ul.ah-stacked-cards{list-style:none;padding:0}.ah-reading-guide .ah-stacked-cards>li{margin:14px 0;padding:20px 24px;border:1px solid #dbe7f1;border-radius:12px;background:#f8fbfe}\n.ah-reading-guide .ah-minor-label{font-weight:700;color:#1e4e64;font-size:18px;margin:20px 0 8px;}\n.ah-reading-guide h3.ah-do,.ah-reading-guide h3.ah-dont{font-size:21px;padding:18px 22px;border-radius:0 10px 10px 0}.ah-reading-guide h3.ah-do{border-left:4px solid #20877e;background:#f0faf8}.ah-reading-guide h3.ah-dont{border-left:4px solid #ae6340;background:#fff7f0}\n.ah-reading-guide p.ah-fact-row{margin:6px 0;padding:12px 18px;background:#f4f8fc;border-left:3px solid #c7dcf0;border-radius:0 8px 8px 0;font-size:17px;line-height:1.65}.ah-reading-guide p.ah-cta a{display:inline-block;padding:12px 20px;border-radius:9px;background:#135faa;color:#fff;text-decoration:none;font-weight:700}.ah-reading-guide p.ah-cta a:hover{background:#104d89}.ah-reading-guide .ah-agency-summary td:first-child{width:38%;color:#184b75}\n.ah-reading-guide details.ah-original-index{padding:16px 20px;border:1px solid #dce7f0;border-radius:10px;margin:24px 0;background:#f8fafc}.ah-reading-guide details.ah-original-index summary{cursor:pointer;font-size:18px;font-weight:700;color:#244f74}.ah-reading-guide details.ah-original-index>ol{margin-top:18px}\n.ah-reading-guide ul.ah-cycle-cards{grid-template-columns:repeat(3,minmax(0,1fr));gap:22px}.ah-reading-guide ul.ah-cycle-cards>li{position:relative;text-align:center;display:flex;align-items:center;justify-content:center;min-height:110px;font-weight:700;color:#20567d}.ah-reading-guide ul.ah-cycle-cards>li:last-child:nth-child(odd){grid-column:auto}.ah-reading-guide ul.ah-cycle-cards>li:not(:last-child)::after{content:\"\u203a\";position:absolute;right:-19px;top:50%;transform:translateY(-50%);font-size:28px;color:#3785b4}@media(max-width:700px){.ah-reading-guide ul.ah-cycle-cards{grid-template-columns:1fr}.ah-reading-guide ul.ah-cycle-cards>li:not(:last-child)::after{right:50%;top:auto;bottom:-25px;transform:rotate(90deg)}}\n.ah-reading-guide .ah-process-10::before{content:\"10\"}\n.ah-reading-guide pre.wp-block-code{background:#122238;color:#eff6ff;border:1px solid #29415f;border-radius:12px;padding:22px 24px;margin:24px 0;overflow-x:auto;font-size:15px;line-height:1.7;white-space:pre;tab-size:4}.ah-reading-guide pre.wp-block-code code{background:transparent;color:inherit;padding:0;white-space:pre;font-family:Consolas,Monaco,monospace}.ah-reading-guide p code{background:#edf3fa;color:#204d78;border-radius:4px;padding:2px 5px}\n.ah-reading-guide ol.ah-steps.ah-start-2{counter-reset:ah-step 1}\n.ah-reading-guide ol.ah-steps.ah-start-3{counter-reset:ah-step 2}\n.ah-reading-guide ol.ah-steps.ah-start-4{counter-reset:ah-step 3}\n.ah-reading-guide ol.ah-steps.ah-start-5{counter-reset:ah-step 4}\n.ah-reading-guide ol.ah-steps.ah-start-6{counter-reset:ah-step 5}\n.ah-reading-guide ol.ah-steps.ah-start-7{counter-reset:ah-step 6}\n.ah-reading-guide ol.ah-steps.ah-start-8{counter-reset:ah-step 7}\n.ah-reading-guide ol.ah-steps.ah-start-9{counter-reset:ah-step 8}\n.ah-reading-guide ol.ah-steps.ah-start-10{counter-reset:ah-step 9}\n.ah-reading-guide ol.ah-steps.ah-start-11{counter-reset:ah-step 10}\n.ah-reading-guide ol.ah-steps.ah-start-12{counter-reset:ah-step 11}\n.ah-reading-guide #ez-toc-container nav{max-height:460px;overflow-y:auto;padding-right:8px}.ah-reading-guide figure.ah-comparison-wide{max-height:650px;overflow:auto;border:1px solid #dae4ee;border-radius:12px}.ah-reading-guide .ah-comparison-wide table{min-width:760px;font-size:15px}.ah-reading-guide .ah-comparison-wide thead th{position:sticky;top:0;background:#183f65;color:#fff;z-index:1}.ah-reading-guide figure.ah-agency-summary tbody th{width:30%;background:#edf4fa;color:#244764;font-weight:700}\n.ah-reading-guide #ez-toc-container a.ez-toc-link::before{content:none!important;display:none!important}\n.ah-reading-guide .ah-comparison-wide thead th,.ah-reading-guide .ah-comparison-wide thead th *{color:#fff!important}\n.ah-reading-guide .ah-equation{display:block;margin:16px 0;padding:16px 20px;border-left:4px solid #2378b9;background:#edf4fa;border-radius:8px;font-size:20px;font-variant-numeric:tabular-nums;overflow-wrap:anywhere}\n.ah-reading-guide .ah-feature-comparison thead th:nth-child(1){width:18%}.ah-reading-guide .ah-feature-comparison thead th:nth-child(2){width:10%}.ah-reading-guide .ah-feature-comparison thead th:nth-child(3){width:50%}.ah-reading-guide .ah-feature-comparison thead th:nth-child(4){width:22%}\n.ah-reading-guide.ah-compact-toc #ez-toc-container ul.ez-toc-list>li>ul>li>ul{display:none}.ah-reading-guide h4.ah-do{color:#276044!important}.ah-reading-guide h4.ah-dont{color:#855218!important}.ah-reading-guide p.ah-tool-bullet{margin:.45em 0!important;padding:.6em 1em;border-radius:8px;background:#f4f7fa;line-height:1.65}\n.ah-reading-guide .ah-glossary-grid{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:20px;margin:30px 0}.ah-reading-guide .ah-glossary-entry{background:#f7f9fc;border:1px solid #dce5ef;border-radius:12px;padding:22px;height:100%;box-sizing:border-box}.ah-reading-guide .ah-glossary-entry h2{font-size:21px!important;margin:0 0 14px!important;line-height:1.4!important;border:0;padding:0}.ah-reading-guide .ah-glossary-entry p{font-size:16px!important;line-height:1.7!important;margin:0!important}@media(max-width:650px){.ah-reading-guide .ah-glossary-grid{grid-template-columns:1fr}}\n.ah-reading-guide .ah-glossary-grid{display:block}.ah-reading-guide .ah-glossary-grid>.wp-block-group__inner-container{display:grid;grid-template-columns:repeat(2,minmax(0,1fr));gap:20px;align-items:stretch}.ah-reading-guide .ah-glossary-entry{height:auto!important;margin:0!important;min-width:0}.ah-reading-guide .ah-glossary-grid>.wp-block-group__inner-container>.wp-block-greenshift-blocks-container{display:flex;min-width:0}.ah-reading-guide .ah-glossary-grid>.wp-block-group__inner-container>.wp-block-greenshift-blocks-container>.ah-glossary-entry{flex:1}@media(max-width:650px){.ah-reading-guide .ah-glossary-grid>.wp-block-group__inner-container{grid-template-columns:1fr}}\n.ah-reading-guide ol.ah-steps.ah-feature-grid{grid-template-columns:repeat(2,minmax(0,1fr))}@media(max-width:700px){.ah-reading-guide ol.ah-steps.ah-feature-grid{grid-template-columns:1fr}}<\/style><noscript><style>.ah-reading-guide [data-aos]{opacity:1!important;visibility:visible!important;transform:none!important}<\/style><\/noscript><p class=\"ah-lead wp-block-paragraph\"><strong><em>Harness the power of Retrieval Augmented Generation (RAG) to create an intelligent document interaction system using FastAPI, Haystack, ChromaDB, and Crawl4AI.<\/em><\/strong><\/p><details id=\"f7a6\" class=\"ah-callout\"><summary style=\"cursor:pointer;font-weight:700\">Table of Contents<\/summary><ol class=\"wp-block-list\"><li>Problem statement<\/li><li>Prerequisites<\/li><li>Project components<\/li><li>Setting Up the Environment<\/li><li>Building the FastAPI Server<\/li><li>Building the Haystack RAG pipeline<\/li><li>Testing the APIs<\/li><li>Conclusion<\/li><li>Optional: Adding Asynchronous Processing with Celery<\/li><\/ol><\/details><hr class=\"wp-block-separator has-alpha-channel-opacity\"\/><h2 id=\"6fd9\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Problem_Statement\"><\/span>Problem Statement<span class=\"ez-toc-section-end\"><\/span><\/h2><p class=\"ah-definition wp-block-paragraph\" id=\"de0d\">The problem statement is to develop a backend system that, provided any URL as input, allows the user to converse about its contents.<\/p><p class=\"wp-block-paragraph\" id=\"df48\">For example, given a news article input or a hosted PDF as input users should be able to ask questions about that PDF or the web article like<\/p><ul class=\"wp-block-list ah-cards\"><li>what is the PDF about?<\/li><li>summarize the article for me in 100 words<\/li><li>Or any particular question about the URL<\/li><\/ul><h2 id=\"dc1e\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Introduction\"><\/span>Introduction<span class=\"ez-toc-section-end\"><\/span><\/h2><p>In the era of information overload, extracting relevant information from vast amounts of data is a significant challenge. Retrieval Augmented Generation (RAG) combines retrieval and generation to provide precise answers from specific documents. In this article, we\u2019ll build a RAG pipeline that allows users to ingest any URL from various sources and interact with them using natural language queries.<\/p><p class=\"wp-block-paragraph\" id=\"bbdd\">We\u2019ll utilize:<\/p><div class=\"wp-block-greenshift-blocks-container gspb_container gspb_container-gsbp-f10880-1\" id=\"gspb_container-id-gsbp-f10880-1\" data-aos=\"fade\" data-aos-easing=\"ease\" data-aos-duration=\"500\" data-aos-once=\"true\"><ul class=\"wp-block-list ah-cards\"><li><a href=\"https:\/\/fastapi.tiangolo.com\/\" target=\"_blank\" rel=\"noreferrer noopener\">FastAPI<\/a>: For building the RESTful API.<\/li><li><a href=\"https:\/\/github.com\/unclecode\/crawl4ai\" target=\"_blank\" rel=\"noreferrer noopener\">Crawl4AI<\/a>: A powerful tool for asynchronous web crawling and data extraction.<\/li><li><a href=\"https:\/\/haystack.deepset.ai\/\" target=\"_blank\" rel=\"noreferrer noopener\">Haystack<\/a>: An open-source framework for building search systems.<\/li><li><a href=\"https:\/\/www.trychroma.com\/\" target=\"_blank\" rel=\"noreferrer noopener\">ChromaDB<\/a>: A vector database for storing embeddings.<\/li><\/ul><\/div><h4 id=\"ef4f\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Optional_Components\"><\/span>Optional Components:<span class=\"ez-toc-section-end\"><\/span><\/h4><p class=\"wp-block-paragraph\" id=\"b59b\"><a href=\"https:\/\/docs.celeryq.dev\/en\/stable\/\" rel=\"noreferrer noopener\" target=\"_blank\">Celery<\/a>&nbsp;and&nbsp;<a href=\"https:\/\/redis.io\/docs\/latest\/develop\/clients\/redis-py\/\" rel=\"noreferrer noopener\" target=\"_blank\">Redis<\/a>: For asynchronous task processing to handle long-running tasks in the background.<\/p><hr class=\"wp-block-separator has-alpha-channel-opacity\"\/><h2 id=\"8bb1\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Prerequisites\"><\/span>Prerequisites<span class=\"ez-toc-section-end\"><\/span><\/h2><p class=\"wp-block-paragraph\" id=\"81bb\">Before diving in, ensure you have the following:<\/p><ul class=\"wp-block-list ah-cards\"><li><a href=\"https:\/\/www.python.org\/downloads\/\" target=\"_blank\" rel=\"noreferrer noopener\">Python 3.8+ installed<\/a>.<\/li><li>Familiarity with Python and FastAPI.<\/li><li>Basic understanding of RAG and how it works.<\/li><\/ul><hr class=\"wp-block-separator has-alpha-channel-opacity\"\/><h2 id=\"5b9b\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Project_Components\"><\/span>Project Components<span class=\"ez-toc-section-end\"><\/span><\/h2><ol class=\"wp-block-list\"><li>FastAPI Server: Exposes two endpoints:<\/li><\/ol><div class=\"wp-block-greenshift-blocks-container gspb_container gspb_container-gsbp-f10880-2\" id=\"gspb_container-id-gsbp-f10880-2\" data-aos=\"fade\" data-aos-easing=\"ease\" data-aos-duration=\"500\" data-aos-once=\"true\"><ul class=\"wp-block-list ah-cards\"><li>Ingest API: Accepts a URL, processes it, and stores the content.<\/li><li>Generate API: Accepts a question and the URL (same as ingestion) retrieves relevant content to generate an answer.<\/li><\/ul><\/div><p class=\"wp-block-paragraph\" id=\"e0e8\">2. Crawl4AI: Simplifies asynchronous web crawling and data extraction.<\/p><p class=\"wp-block-paragraph\" id=\"0cc7\">3. Haystack with ChromaDB: Stores and retrieves document embeddings.<\/p><hr class=\"wp-block-separator has-alpha-channel-opacity\"\/><h2 id=\"d9d1\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Setting_Up_the_Environment\"><\/span>Setting Up the Environment<span class=\"ez-toc-section-end\"><\/span><\/h2><p class=\"wp-block-paragraph\" id=\"0929\">We\u2019ll set up the project from scratch using the provided instructions.<\/p><h3 id=\"c8bc\" class=\"wp-block-heading has-medium-font-size\"><span class=\"ez-toc-section\" id=\"1_Create_a_Project_Directory\"><\/span>1. Create a Project Directory<span class=\"ez-toc-section-end\"><\/span><\/h3><p class=\"wp-block-paragraph\" id=\"e7af\">Create a new directory for your project and navigate into it.<\/p><h3 id=\"5441\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"2_Create_a_Virtual_Environment\"><\/span>2. Create a Virtual Environment<span class=\"ez-toc-section-end\"><\/span><\/h3><p class=\"wp-block-paragraph\" id=\"9c5e\">Create and activate a virtual environment to manage project dependencies<\/p><h4 id=\"for-windows\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"For_Windows\"><\/span>For Windows:<span class=\"ez-toc-section-end\"><\/span><\/h4><pre class=\"wp-block-code\"><code>python -m venv venv<\/code><\/pre><pre class=\"wp-block-code\"><code>venv\\Scripts\\activate<\/code><\/pre><h4 id=\"for-macos-linux\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"For_macOSLinux\"><\/span>For macOS\/Linux:<span class=\"ez-toc-section-end\"><\/span><\/h4><pre class=\"wp-block-code\"><code>python3 -m venv venv<\/code><\/pre><pre class=\"wp-block-code\"><code>source venv\/bin\/activate<\/code><\/pre><h3 id=\"a650\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"3_Install_Dependencies\"><\/span>3. Install Dependencies<span class=\"ez-toc-section-end\"><\/span><\/h3><p class=\"wp-block-paragraph\" id=\"b804\">We\u2019ll need several Python packages.<\/p><p class=\"wp-block-paragraph\" id=\"81a4\">Install them using pip:<\/p><pre class=\"wp-block-code\"><code>pip install fastapi uvicorn&#091;standard] requests crawl4ai farm-haystack chromadb chroma-haystack haystack-ai ollama-haystack python-multipart<\/code><\/pre><p class=\"wp-block-paragraph\"><em>Alternatively, you can create a requirements.txt file with all dependencies and install them with<\/em><\/p><h4 id=\"requirements-txt\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"requirementstxt\"><\/span><strong>requirements.txt<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h4><pre class=\"wp-block-code\"><code>haystack-aiollama-haystackchroma-haystackpython-multipartfastapiuvicorncrawl4ai&#091;sync]playwrightpypdfmarkdown-it-pymdit_plainfiletypeCelery&#091;redis] # Optional<\/code><\/pre><p class=\"wp-block-paragraph\">Install the dependencies:<\/p><pre class=\"wp-block-code\"><code>pip install -r requirements.txt<\/code><\/pre><h3 id=\"4-install-playwright-and-its-dependencies\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"4_Install_Playwright_and_its_dependencies\"><\/span>4. Install Playwright and its dependencies<span class=\"ez-toc-section-end\"><\/span><\/h3><pre class=\"wp-block-code\"><code>playwright installplaywright install-deps<\/code><\/pre><h2 id=\"building-the-fastapi-server\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Building_the_FastAPI_Server\"><\/span>Building the FastAPI Server<span class=\"ez-toc-section-end\"><\/span><\/h2><p class=\"wp-block-paragraph\"><em>Create a file named main.py and set up the FastAPI app.<\/em><\/p><pre class=\"wp-block-code\"><code>from fastapi import FastAPIapp = FastAPI() # initialize the Fastapi app<\/code><\/pre><h3 id=\"1-handling-different-url-types\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"1_Handling_Different_URL_Types\"><\/span>1. Handling Different URL Types<span class=\"ez-toc-section-end\"><\/span><\/h3><p class=\"wp-block-paragraph\">First let <a  data-ilj-link-preview=\"true\"  data-featured-image=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2022\/06\/selling-on-amazon-mexico-300x225.jpg\"  data-excerpt=\"Mexico&#039;s eCommerce boom presents huge opportunities for Amazon sellers. This guide explores the thriving Mexican online market, Amazon.mx&#039;s growth, and key considerations for successfully selling your products to millions of Mexican consumers. Learn how to navigate logistics, expand your reach, and capitalize on this expanding market.\" href=\"https:\/\/www.aihello.com\/resources\/blog\/how-to-launch-an-amazon-mexico-business\/\">us<\/a> create a utility that given a URL as input, classifies it into two basic categories.<\/p><ol class=\"wp-block-list ah-cards\"><li>File: .pdf, .txt, .md, etc<\/li><li>Web-page or article<\/li><\/ol><p class=\"wp-block-paragraph\">We\u2019ll use the requests and mime-types libraries to identify file types based on the URL\u2019s content type.<\/p><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"786\" height=\"757\" title=\"how does amazon use business analytics | how to start selling on amazon uk | what is e tailing\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-5.png\" alt=\"\" class=\"wp-image-10888\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-5.png 786w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-5-300x289.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-5-768x740.png 768w\" sizes=\"auto, (max-width: 786px) 100vw, 786px\" \/><\/figure><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"767\" height=\"663\" title=\"how do lightning deals work on amazon | do i need a company to sell on amazon germany | how much are amazon sponsored ads\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-6.png\" alt=\"\" class=\"wp-image-10890\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-6.png 767w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-6-300x259.png 300w\" sizes=\"auto, (max-width: 767px) 100vw, 767px\" \/><\/figure><p class=\"wp-block-paragraph\" id=\"0b38\">This function returns the URL&#8217;s mime type, representing what type of URL it is.<\/p><p class=\"wp-block-paragraph\" id=\"601b\">It returns \u201carticle\u201d if it is a web page or web article and the file extension if it is a document such as .pdf for PDF files, .txt for text files, .md markdown, etc.<\/p><h3 id=\"2-crawling-web-articles-with-crawl4ai\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"2_Crawling_Web_Articles_with_Crawl4AI\"><\/span>2. Crawling Web Articles with Crawl4AI<span class=\"ez-toc-section-end\"><\/span><\/h3><h4 id=\"what-is-crawl4ai\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"What_is_Crawl4AI\"><\/span>What is Crawl4AI?<span class=\"ez-toc-section-end\"><\/span><\/h4><p class=\"ah-definition wp-block-paragraph\" id=\"7eaf\">Crawl4AI simplifies asynchronous web crawling and data extraction, making it accessible for large language models (LLMs) and AI applications. It handles the complexities of HTTP requests, parsing, and data extraction, providing a straightforward interface for developers.<\/p><h4 id=\"65d5\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Using_Crawl4AI_to_Scrape_Web_Articles\"><\/span>Using Crawl4AI to Scrape Web Articles<span class=\"ez-toc-section-end\"><\/span><\/h4><p class=\"wp-block-paragraph\" id=\"56b9\">We will create an asynchronous function that given a URL scrapes it using the crawl4ai python package.<\/p><p class=\"wp-block-paragraph\" id=\"8405\">Check out the&nbsp;<a href=\"https:\/\/crawl4ai.com\/mkdocs\/basic\/installation\/\" rel=\"noreferrer noopener\" target=\"_blank\">crawl4ai documentation<\/a>&nbsp;if you need help with it.<\/p><p class=\"wp-block-paragraph\"><strong>Example Usage:<\/strong><\/p><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"848\" height=\"887\" title=\"what does acos stand for | how to get an amazon account manager | what is amazon vine and how does it work\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-7.png\" alt=\"\" class=\"wp-image-10891\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-7.png 848w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-7-287x300.png 287w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-7-768x803.png 768w\" sizes=\"auto, (max-width: 848px) 100vw, 848px\" \/><\/figure><p class=\"wp-block-paragraph\"><br><br>We will make use of the AsyncWebCrawler to pass another class object LLMExtractionStrategy from crawl4ai to it.<\/p><p class=\"wp-block-paragraph\" id=\"8157\">The LLMExtractionStrategy takes the following important parameters as input,<\/p><ol class=\"wp-block-list ah-cards\"><li><strong>provider<\/strong>&nbsp;\u2014 The <a  data-ilj-link-preview=\"true\"  data-featured-image=\"\"  data-excerpt=\"Understanding tokenization: Learn why this NLP process is crucial for language models (LLMs). Discover how text is converted for AI and explore different tokenizer types.\" href=\"https:\/\/www.aihello.com\/resources\/blog\/tokenization-and-its-application\/\">LLM<\/a> provided which you want to use, for this tutorial, we will use&nbsp;<a href=\"https:\/\/ollama.com\/\" target=\"_blank\" rel=\"noreferrer noopener\"><strong>Ollama<\/strong><\/a>, which is a locally hosted LLM. Alternatively, you can also use other LLM providers as suggested on the&nbsp;<a href=\"https:\/\/docs.litellm.ai\/docs\/providers\" target=\"_blank\" rel=\"noreferrer noopener\"><strong>LiteLLM<\/strong><\/a>&nbsp;docs<\/li><li><strong>base_url<\/strong>&nbsp;\u2014 The api_base where you have your Ollama hosted, if you have it hosted locally it should be something like&nbsp;<strong>localhost:11434<\/strong><\/li><li><strong>instruction<\/strong>: The prompt or instruction you want to send to the LLM while scraping the page<\/li><li><strong>api_token<\/strong>&nbsp;\u2014 the api_key if using hosted LLM providers like Openai or Gemini.<\/li><\/ol><p class=\"wp-block-paragraph\" id=\"867a\">You can check out all other parameters that it takes in its&nbsp;<a href=\"https:\/\/crawl4ai.com\/mkdocs\/\" rel=\"noreferrer noopener\" target=\"_blank\"><strong>documentation<\/strong><\/a>.<\/p><h3 id=\"848b\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"3_Implementing_the_Ingest_Endpoint\"><\/span><strong>3. Implementing the Ingest Endpoint<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3><p class=\"wp-block-paragraph\" id=\"893b\">Now, Let us create the ingest endpoint first, which accepts the URL as an input, determines its category using the identify_url function that we created earlier, and passes it to two different ingestion functions namely ingest_url and ingest_file that we will be creating soon.<\/p><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"797\" height=\"832\" title=\"what is kindle direct publishing | can i sell on amazon if i live in australia | what partners offer end-to-end amazon marketing services?\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-8.png\" alt=\"\" class=\"wp-image-10892\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-8.png 797w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-8-287x300.png 287w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-8-768x802.png 768w\" sizes=\"auto, (max-width: 797px) 100vw, 797px\" \/><\/figure><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"738\" height=\"702\" title=\"how amazon advertising works | how to improve amazon sales rank | how to sell on amazon usa from canada\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-9.png\" alt=\"\" class=\"wp-image-10893\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-9.png 738w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-9-300x285.png 300w\" sizes=\"auto, (max-width: 738px) 100vw, 738px\" \/><\/figure><h3 id=\"4781\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"4_Implementing_the_Generate_API\"><\/span><strong>4. Implementing the Generate API<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3><p class=\"wp-block-paragraph\" id=\"25dd\">Now, we shall also create an endpoint that given the URL and a question, classifies it using the&nbsp;<strong>indetify_url<\/strong>&nbsp;function and routes it to different two functions namely&nbsp;<strong>get_url_result<\/strong>&nbsp;and&nbsp;<strong>get_file_result<\/strong>&nbsp;to generate the response for that question. We will also create these functions soon.<\/p><p class=\"wp-block-paragraph\" id=\"25dd\"><\/p><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"790\" height=\"817\" title=\"how to lower acos | how much is advertising on amazon | how to be a top seller on amazon\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/generate-api-endpoint.png\" alt=\"\" class=\"wp-image-10894\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/generate-api-endpoint.png 790w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/generate-api-endpoint-290x300.png 290w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/generate-api-endpoint-768x794.png 768w\" sizes=\"auto, (max-width: 790px) 100vw, 790px\" \/><\/figure><p class=\"wp-block-paragraph\" id=\"25dd\"><\/p><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"785\" height=\"806\" title=\"how to remove reviews from amazon | what are sponsored products on amazon | how many keyword groups can a keyword be associated with\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-11.png\" alt=\"\" class=\"wp-image-10895\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-11.png 785w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-11-292x300.png 292w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-11-768x789.png 768w\" sizes=\"auto, (max-width: 785px) 100vw, 785px\" \/><\/figure><p class=\"wp-block-paragraph\" id=\"25dd\"><\/p><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"778\" height=\"537\" title=\"how to reduce amazon acos | how to join amazon vine | how to rank higher on amazon\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-12.png\" alt=\"\" class=\"wp-image-10896\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-12.png 778w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-12-300x207.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-12-768x530.png 768w\" sizes=\"auto, (max-width: 778px) 100vw, 778px\" \/><\/figure><p class=\"wp-block-paragraph\" id=\"25dd\"><\/p><h2 id=\"384b\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Building_the_HayStack_RAG_pipeline\"><\/span>Building the HayStack RAG pipeline<span class=\"ez-toc-section-end\"><\/span><\/h2><p class=\"wp-block-paragraph\" id=\"0952\">Let us implement the logic for the ingestion and result generation functions.<\/p><p class=\"wp-block-paragraph\" id=\"066e\">First, let us set up the LLM Backend, we will be using the same LLM that we used for crawling for chat purposes.<\/p><p class=\"ah-callout wp-block-paragraph\" id=\"9584\">Note that, since RAG requires two kinds of models, one for conversations and one to generate embeddings, we will require two and install them on our llama before using it.<\/p><p class=\"wp-block-paragraph\" id=\"96b8\">Here, I am using llama3.2 for chat and nomic-embed-text for the generation of embedding.<\/p><p class=\"ah-callout wp-block-paragraph\" id=\"baa7\">Note that you don\u2019t need to worry about how embedding or the chat is being handled, you just have to pass the model names, and Haystack will take care of it all.<\/p><h3 id=\"dcc9\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Setup_the_LLM_Backend_and_Prompt\"><\/span><strong>Setup the LLM Backend and Promp<\/strong>t<span class=\"ez-toc-section-end\"><\/span><\/h3><p class=\"wp-block-paragraph\" id=\"dcc9\"><\/p><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"780\" height=\"848\" title=\"how is acos calculated | how to do business with amazon india | how to grow sales on amazon\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-13.png\" alt=\"\" class=\"wp-image-10897\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-13.png 780w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-13-276x300.png 276w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-13-768x835.png 768w\" sizes=\"auto, (max-width: 780px) 100vw, 780px\" \/><\/figure><p class=\"wp-block-paragraph\" id=\"dcc9\"><\/p><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"782\" height=\"842\" title=\"how to get amazon best seller badge | what is an amazon lightning deal | how to start selling on amazon canada\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-14.png\" alt=\"\" class=\"wp-image-10898\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-14.png 782w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-14-279x300.png 279w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-14-768x827.png 768w\" sizes=\"auto, (max-width: 782px) 100vw, 782px\" \/><\/figure><p class=\"wp-block-paragraph\" id=\"dcc9\"><\/p><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"787\" height=\"816\" title=\"how to use brand analytics amazon | is there amazon in saudi arabia | how to find competitors adwords keywords\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-15.png\" alt=\"\" class=\"wp-image-10899\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-15.png 787w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-15-289x300.png 289w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-15-768x796.png 768w\" sizes=\"auto, (max-width: 787px) 100vw, 787px\" \/><\/figure><p class=\"wp-block-paragraph\" id=\"dcc9\"><\/p><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"786\" height=\"500\" title=\"what software integrates with amazon to improve account health? | how to get paid by amazon | how to optimize an amazon listing\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-16.png\" alt=\"\" class=\"wp-image-10900\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-16.png 786w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-16-300x191.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-16-768x489.png 768w\" sizes=\"auto, (max-width: 786px) 100vw, 786px\" \/><\/figure><p class=\"ah-callout wp-block-paragraph\" id=\"dcc9\"><em>Note: Replace Your_Ollama_URL with your ollama URL<\/em><\/p><p class=\"wp-block-paragraph\" id=\"b224\">Now, let us build the necessary functionality to process the documents and URLs, separately.<\/p><h3 id=\"processing-urls\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Processing_URLs\"><\/span><strong>Processing URLs<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"778\" height=\"743\" title=\"how to reduce amazon fba fees | which of the following is a requirement for amazon sellers to set up sponsored brands campaigns | how to source products to sell on amazon fba\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-17.png\" alt=\"\" class=\"wp-image-10901\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-17.png 778w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-17-300x287.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-17-768x733.png 768w\" sizes=\"auto, (max-width: 778px) 100vw, 778px\" \/><\/figure><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"782\" height=\"682\" title=\"what&apos;s the best selling item on amazon | how to reduce acos amazon ppc | what is amazon.com.mx\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-18.png\" alt=\"\" class=\"wp-image-10902\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-18.png 782w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-18-300x262.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-18-768x670.png 768w\" sizes=\"auto, (max-width: 782px) 100vw, 782px\" \/><\/figure><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"785\" height=\"838\" title=\"how to sell on amazon ksa | what is amazon kindle direct publishing | does amazon advertise\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-19.png\" alt=\"\" class=\"wp-image-10903\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-19.png 785w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-19-281x300.png 281w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-19-768x820.png 768w\" sizes=\"auto, (max-width: 785px) 100vw, 785px\" \/><\/figure><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"785\" height=\"911\" title=\"is fulfillment by amazon worth it | how to optimize amazon product listings | how to boost your amazon listing\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/query-pipeline-processing-png.png\" alt=\"\" class=\"wp-image-10904\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/query-pipeline-processing-png.png 785w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/query-pipeline-processing-png-259x300.png 259w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/query-pipeline-processing-png-768x891.png 768w\" sizes=\"auto, (max-width: 785px) 100vw, 785px\" \/><\/figure><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"786\" height=\"230\" title=\"where to buy products to sell on amazon | how to optimize amazon ppc campaigns | how to deal with fake amazon reviews\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-21.png\" alt=\"\" class=\"wp-image-10905\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-21.png 786w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-21-300x88.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-21-768x225.png 768w\" sizes=\"auto, (max-width: 786px) 100vw, 786px\" \/><\/figure><hr class=\"wp-block-separator has-alpha-channel-opacity\"\/><h3 id=\"processing-documents\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Processing_Documents\"><\/span><strong>Processing Documents<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"792\" height=\"446\" title=\"how to optimize amazon ads | how to optimize your amazon listing | how do you sell books to amazon\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-22.png\" alt=\"\" class=\"wp-image-10906\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-22.png 792w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-22-300x169.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-22-768x432.png 768w\" sizes=\"auto, (max-width: 792px) 100vw, 792px\" \/><\/figure><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"783\" height=\"822\" title=\"does mexico have amazon | how to check competitors keywords | how can i sell on amazon\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-23.png\" alt=\"\" class=\"wp-image-10907\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-23.png 783w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-23-286x300.png 286w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-23-768x806.png 768w\" sizes=\"auto, (max-width: 783px) 100vw, 783px\" \/><\/figure><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"788\" height=\"647\" title=\"is ams amazon | how does amazon get their products | what is trending on amazon right now\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-24.png\" alt=\"\" class=\"wp-image-10908\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-24.png 788w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-24-300x246.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-24-768x631.png 768w\" sizes=\"auto, (max-width: 788px) 100vw, 788px\" \/><\/figure><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"786\" height=\"815\" title=\"what is listing optimization in amazon | who are the top amazon advertising and strategy agencies? | how to get sales on amazon\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-25.png\" alt=\"\" class=\"wp-image-10909\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-25.png 786w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-25-289x300.png 289w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-25-768x796.png 768w\" sizes=\"auto, (max-width: 786px) 100vw, 786px\" \/><\/figure><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"780\" height=\"588\" title=\"how many keywords does amazon allow | how to choose keywords for amazon | how to sell products on amazon germany\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-26.png\" alt=\"\" class=\"wp-image-10910\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-26.png 780w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-26-300x226.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-26-768x579.png 768w\" sizes=\"auto, (max-width: 780px) 100vw, 780px\" \/><\/figure><h2 id=\"testing-the-apis\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Testing_the_APIs\"><\/span><strong>Testing the APIs<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h2><h3 id=\"starting-the-fastapi-application\" class=\"wp-block-heading ah-process-heading ah-process-1\"><span class=\"ez-toc-section\" id=\"Starting_the_FastAPI_Application\"><\/span><strong>Starting the FastAPI Application<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3><p class=\"wp-block-paragraph\" id=\"9a92\">In your terminal or command prompt, navigate to your project directory and activate the virtual environment.<\/p><p class=\"wp-block-paragraph\" id=\"bb24\">Then run:<\/p><pre class=\"wp-block-code\"><code>uvicorn main:app - reload<\/code><\/pre><h3 id=\"60d4\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"2_Testing_with_API_Documentation\"><\/span><strong>2. Testing with API Documentation<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h3><p class=\"wp-block-paragraph\" id=\"6e82\">FastAPI provides interactive API documentation at&nbsp;<a href=\"http:\/\/localhost:8000\/docs.\" rel=\"noreferrer noopener\" target=\"_blank\"><strong>http:\/\/localhost:8000\/docs<\/strong>.<\/a><\/p><p class=\"wp-block-paragraph\" id=\"35a1\">You can test your APIs directly from this interface. <a  data-ilj-link-preview=\"true\"  data-featured-image=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2020\/04\/images-1-300x147.jpg\"  data-excerpt=\"Optimize your Amazon seller journey! Discover how FBA streamlines shipping, boosts seller ratings, and leverages customer feedback for online success.\" href=\"https:\/\/www.aihello.com\/resources\/blog\/key-terms-you-need-to-know-when-you-start-out-as-a-seller\/\">You will<\/a> see the swagger-UI like this.<\/p><h4 id=\"64c1\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Testing_the_Ingest_API\"><\/span>Testing the Ingest API<span class=\"ez-toc-section-end\"><\/span><\/h4><ol class=\"wp-block-list ah-steps\"><li>Open your web browser and navigate to&nbsp;<a href=\"http:\/\/localhost:8000\/docs.\" target=\"_blank\" rel=\"noreferrer noopener\"><strong>http:\/\/localhost:8000\/docs<\/strong>.<\/a><\/li><li>Find the&nbsp;<strong>\/ingest&nbsp;<\/strong>endpoint and expand it.<\/li><li>Click on the \u201cTry it out\u201d button.<\/li><\/ol><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"828\" height=\"424\" title=\"how to edit amazon advertising campaign | what does roas stand for | how to list books on amazon\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-2.png\" alt=\"\" class=\"wp-image-10884\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-2.png 828w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-2-300x154.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-2-768x393.png 768w\" sizes=\"auto, (max-width: 828px) 100vw, 828px\" \/><\/figure><ol class=\"wp-block-list ah-steps ah-start-4\"><li>Enter a URL to ingest (e.g., a web article or a PDF link).<\/li><li>Click \u201cExecute\u201d<\/li><li>You\u2019ll receive a confirmation message if the content is ingested successfully.<\/li><\/ol><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"828\" height=\"261\" title=\"how to sell on amazon europe | can i make money on amazon | how to sell on amazon usa from canada\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-1.png\" alt=\"\" class=\"wp-image-10883\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-1.png 828w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-1-300x95.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-1-768x242.png 768w\" sizes=\"auto, (max-width: 828px) 100vw, 828px\" \/><\/figure><h4 id=\"464c\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Testing_the_Generate_API\"><\/span>Testing the Generate API<span class=\"ez-toc-section-end\"><\/span><\/h4><ol class=\"wp-block-list ah-steps\"><li>Once the content is ingested, find the&nbsp;<strong>\/generate<\/strong>&nbsp;endpoint.<\/li><li>Click on \u201cTry it out\u201d<\/li><\/ol><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"828\" height=\"356\" title=\"what is amazon optimization | what does lightning deal mean on amazon | do i need a company to sell on amazon germany\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-3.png\" alt=\"\" class=\"wp-image-10885\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-3.png 828w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-3-300x129.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-3-768x330.png 768w\" sizes=\"auto, (max-width: 828px) 100vw, 828px\" \/><\/figure><ol class=\"wp-block-list ah-steps ah-start-3\"><li>Enter a question and a URL related to the content of the ingested document.<\/li><li>Click \u201cExecute\u201d to receive answers from the system.<\/li><\/ol><hr class=\"wp-block-separator has-alpha-channel-opacity\"\/><h2 id=\"972b\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Conclusion\"><\/span>Conclusion<span class=\"ez-toc-section-end\"><\/span><\/h2><p class=\"wp-block-paragraph\" id=\"39e7\">We\u2019ve built a robust RAG pipeline that:<\/p><div class=\"wp-block-greenshift-blocks-container gspb_container gspb_container-gsbp-f10880-3\" id=\"gspb_container-id-gsbp-f10880-3\" data-aos=\"fade\" data-aos-easing=\"ease\" data-aos-duration=\"500\" data-aos-once=\"true\"><ul class=\"wp-block-list ah-cards\"><li>Accepts various document types (web articles, PDFs, text files).<\/li><li>Uses Crawl4AI to simplify web crawling and data extraction.<\/li><li>Allows users to interact with the content using natural language queries through FastAPI.<\/li><li>Utilizes Haystack and ChromaDB for efficient storage and retrieval of document embeddings.<\/li><\/ul><\/div><p class=\"wp-block-paragraph\" id=\"9337\">This system can be extended to support more features like authentication, additional file types, and enhanced error handling. By integrating these technologies, you\u2019ve created a scalable and efficient pipeline for document ingestion and interaction.<\/p><h2 id=\"0d85\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Optional_Adding_Asynchronous_Processing_with_Celery\"><\/span>Optional: Adding Asynchronous Processing with Celery<span class=\"ez-toc-section-end\"><\/span><\/h2><p class=\"wp-block-paragraph\" id=\"28b1\">For long-running tasks like ingesting large documents or crawling extensive web content, it\u2019s beneficial to handle these tasks asynchronously. This ensures that your API remains responsive and doesn\u2019t time out.<\/p><h3 id=\"4b89\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"What_are_Celery_and_Redis\"><\/span>What are Celery and Redis?<span class=\"ez-toc-section-end\"><\/span><\/h3><div class=\"wp-block-greenshift-blocks-container gspb_container gspb_container-gsbp-f10880-4\" id=\"gspb_container-id-gsbp-f10880-4\" data-aos=\"fade\" data-aos-easing=\"ease\" data-aos-duration=\"500\" data-aos-once=\"true\"><ul class=\"wp-block-list ah-cards\"><li>Celery: An asynchronous task queue\/job queue based on distributed message passing.<\/li><li>Redis: An in-memory data structure store used as a database, cache, and message broker.<\/li><\/ul><\/div><h3 id=\"74a2\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Implementing_Asynchronous_Ingestion\"><\/span>Implementing Asynchronous Ingestion<span class=\"ez-toc-section-end\"><\/span><\/h3><h4 id=\"install-celery-and-redis\" class=\"wp-block-heading ah-process-heading ah-process-1\"><span class=\"ez-toc-section\" id=\"Install_Celery_and_Redis\"><\/span><strong>Install Celery and Redis<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h4><pre class=\"wp-block-code\"><code>pip install celery&#091;redis]<\/code><\/pre><p class=\"ah-callout wp-block-paragraph\">Ensure that Redis is running on your system. If not, install and start Redis according to your operating system\u2019s instructions.<\/p><h4 id=\"2-create-a-celery-worker\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"2_Create_a_Celery_Worker\"><\/span><strong>2. Create a Celery Worker<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h4><p class=\"wp-block-paragraph\">Create a file named&nbsp;<strong>celery_worker.py<\/strong><\/p><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"785\" height=\"652\" title=\"how to rank on walmart | how to sell prime on amazon | what&apos;s the best seller on amazon\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-27.png\" alt=\"\" class=\"wp-image-10911\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-27.png 785w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-27-300x249.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-27-768x638.png 768w\" sizes=\"auto, (max-width: 785px) 100vw, 785px\" \/><\/figure><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"785\" height=\"890\" title=\"how to earn money on amazon | who are the top amazon advertising service providers? | how much to advertise on amazon\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-28.png\" alt=\"\" class=\"wp-image-10912\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-28.png 785w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-28-265x300.png 265w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-28-768x871.png 768w\" sizes=\"auto, (max-width: 785px) 100vw, 785px\" \/><\/figure><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"781\" height=\"228\" title=\"how does amazon get their products | how to become an amazon best seller | how to earn money in amazon\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-29.png\" alt=\"\" class=\"wp-image-10913\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-29.png 781w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-29-300x88.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-29-768x224.png 768w\" sizes=\"auto, (max-width: 781px) 100vw, 781px\" \/><\/figure><h4 id=\"3-modify-the-ingest-endpoint\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"3_Modify_the_Ingest_Endpoint\"><\/span><strong>3. Modify the Ingest Endpoint<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h4><p class=\"wp-block-paragraph\">In&nbsp;<strong>main.py<\/strong>, update the ingest endpoint to use Celery.<\/p><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"791\" height=\"758\" title=\"how to list on amazon australia | how to sell on amazon germany from uk | how to source products to sell on amazon\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/api-ingestion-endpoint-png.png\" alt=\"\" class=\"wp-image-10914\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/api-ingestion-endpoint-png.png 791w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/api-ingestion-endpoint-png-300x287.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/api-ingestion-endpoint-png-768x736.png 768w\" sizes=\"auto, (max-width: 791px) 100vw, 791px\" \/><\/figure><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"777\" height=\"377\" title=\"how to do seo on amazon | how does amazon search engine work | is amazon fba worth it 2021\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-31.png\" alt=\"\" class=\"wp-image-10915\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-31.png 777w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-31-300x146.png 300w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/image-31-768x373.png 768w\" sizes=\"auto, (max-width: 777px) 100vw, 777px\" \/><\/figure><figure class=\"wp-block-image size-full\"><img loading=\"lazy\" decoding=\"async\" width=\"781\" height=\"921\" title=\"what is ppc for amazon | how do people make money on amazon | how to calculate amazon conversion rate\" src=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/building-a-rag-pipeline-with-fastapi-haystack-and-chromadb-for-urls-in-python.png\" alt=\"\" class=\"wp-image-10916\" srcset=\"https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/building-a-rag-pipeline-with-fastapi-haystack-and-chromadb-for-urls-in-python.png 781w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/building-a-rag-pipeline-with-fastapi-haystack-and-chromadb-for-urls-in-python-254x300.png 254w, https:\/\/www.aihello.com\/resources\/wp-content\/uploads\/2024\/12\/building-a-rag-pipeline-with-fastapi-haystack-and-chromadb-for-urls-in-python-768x906.png 768w\" sizes=\"auto, (max-width: 781px) 100vw, 781px\" \/><\/figure><h4 id=\"19bd\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"4_Starting_the_Celery_Worker\"><\/span><strong>4. Starting the Celery Worker<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h4><p class=\"wp-block-paragraph\" id=\"5277\">In a new terminal or command prompt window, navigate to your project directory and activate the virtual environment.<\/p><p class=\"wp-block-paragraph\" id=\"0cb4\">Then run:<\/p><pre class=\"wp-block-code\"><code>celery -A celery_worker.celery_app worker --loglevel=info<\/code><\/pre><h4 id=\"75b9\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"5_Testing_Asynchronous_Ingestion\"><\/span>5.&nbsp;<strong>Testing Asynchronous Ingestion<\/strong><span class=\"ez-toc-section-end\"><\/span><\/h4><p class=\"wp-block-paragraph\" id=\"f171\">Follow the same steps in the Testing the APIs section, but now the ingestion will run in the background. You can check the status of your ingestion task using the \/tasks\/{task_id} endpoint.<\/p><hr class=\"wp-block-separator has-alpha-channel-opacity\"\/><h2 id=\"5135\" class=\"wp-block-heading\"><span class=\"ez-toc-section\" id=\"Additional_Resources\"><\/span>Additional Resources<span class=\"ez-toc-section-end\"><\/span><\/h2><ul class=\"wp-block-list\"><li><a href=\"https:\/\/crawl4ai.com\/mkdocs\/\" target=\"_blank\" rel=\"noreferrer noopener\">Crawl4AI Documentation<\/a>: Learn more about how to use Crawl4AI for web crawling and data extraction.<\/li><li><a href=\"https:\/\/fastapi.tiangolo.com\/\" target=\"_blank\" rel=\"noreferrer noopener\">FastAPI Documentation<\/a>: Learn more about building APIs with FastAPI.<\/li><li><a href=\"https:\/\/fastapi.tiangolo.com\/\" target=\"_blank\" rel=\"noreferrer noopener\">Haystack Documentation<\/a>: Explore advanced features of Haystack.<\/li><li><a href=\"https:\/\/docs.trychroma.com\/\" target=\"_blank\" rel=\"noreferrer noopener\">ChromaDB Documentation<\/a>: Understand how to manage vector embeddings.<\/li><li><a href=\"https:\/\/docs.celeryq.dev\/en\/stable\/\" target=\"_blank\" rel=\"noreferrer noopener\">Celery Documentation<\/a>: Dive into asynchronous task processing.<\/li><li><a href=\"https:\/\/redis.io\/docs\/latest\/develop\/clients\/redis-py\/\" target=\"_blank\" rel=\"noreferrer noopener\">Redis Documentation<\/a>: Learn about in-memory data structures and caching.<\/li><\/ul><p class=\"wp-block-paragraph\"><strong>Happy coding! If you have any questions or suggestions, feel free to reach out or leave a comment.<\/strong><\/p><p class=\"wp-block-paragraph\"><\/p><\/div><\/div>","protected":false},"excerpt":{"rendered":"<p>Harness the power of Retrieval Augmented Generation (RAG) to create an intelligent document interaction system using FastAPI, Haystack, ChromaDB, and Crawl4AI.Table of ContentsProblem statementPrerequisitesProject componentsSetting Up the EnvironmentBuilding the FastAPI&#8230;<\/p>\n","protected":false},"author":32,"featured_media":0,"comment_status":"closed","ping_status":"closed","sticky":false,"template":"","format":"standard","meta":{"_gspb_post_css":".gspb_container-id-gsbp-f10880-1,.gspb_container-id-gsbp-f10880-2,.gspb_container-id-gsbp-f10880-3,.gspb_container-id-gsbp-f10880-4{flex-direction:column;box-sizing:border-box}#gspb_container-id-gsbp-f10880-1.gspb_container>p:last-of-type,#gspb_container-id-gsbp-f10880-2.gspb_container>p:last-of-type,#gspb_container-id-gsbp-f10880-3.gspb_container>p:last-of-type,#gspb_container-id-gsbp-f10880-4.gspb_container>p:last-of-type{margin-bottom:0}#gspb_container-id-gsbp-f10880-1.gspb_container.aos-animate,#gspb_container-id-gsbp-f10880-1.gspb_container[data-gs-aos],#gspb_container-id-gsbp-f10880-2.gspb_container.aos-animate,#gspb_container-id-gsbp-f10880-2.gspb_container[data-gs-aos],#gspb_container-id-gsbp-f10880-3.gspb_container.aos-animate,#gspb_container-id-gsbp-f10880-3.gspb_container[data-gs-aos],#gspb_container-id-gsbp-f10880-4.gspb_container.aos-animate,#gspb_container-id-gsbp-f10880-4.gspb_container[data-gs-aos]{opacity:1;transform:translateZ(0)}#gspb_container-id-gsbp-f10880-1.gspb_container,#gspb_container-id-gsbp-f10880-2.gspb_container,#gspb_container-id-gsbp-f10880-3.gspb_container,#gspb_container-id-gsbp-f10880-4.gspb_container{position:relative;transition-duration:.5s;transition-timing-function:var(--gs-root-animation-easing, cubic-bezier(0.42, 0, 0.58, 1));opacity:var(--gs-root-animation-opacity, 0);transition-property:opacity,transform,filter;box-sizing:border-box}","footnotes":""},"categories":[1],"tags":[561],"republish_tag":[],"class_list":["post-10880","post","type-post","status-publish","format-standard","hentry","category-advertising","tag-ai-machine-learning"],"_links":{"self":[{"href":"https:\/\/www.aihello.com\/resources\/wp-json\/wp\/v2\/posts\/10880","targetHints":{"allow":["GET"]}}],"collection":[{"href":"https:\/\/www.aihello.com\/resources\/wp-json\/wp\/v2\/posts"}],"about":[{"href":"https:\/\/www.aihello.com\/resources\/wp-json\/wp\/v2\/types\/post"}],"author":[{"embeddable":true,"href":"https:\/\/www.aihello.com\/resources\/wp-json\/wp\/v2\/users\/32"}],"replies":[{"embeddable":true,"href":"https:\/\/www.aihello.com\/resources\/wp-json\/wp\/v2\/comments?post=10880"}],"version-history":[{"count":1,"href":"https:\/\/www.aihello.com\/resources\/wp-json\/wp\/v2\/posts\/10880\/revisions"}],"predecessor-version":[{"id":14248,"href":"https:\/\/www.aihello.com\/resources\/wp-json\/wp\/v2\/posts\/10880\/revisions\/14248"}],"wp:attachment":[{"href":"https:\/\/www.aihello.com\/resources\/wp-json\/wp\/v2\/media?parent=10880"}],"wp:term":[{"taxonomy":"category","embeddable":true,"href":"https:\/\/www.aihello.com\/resources\/wp-json\/wp\/v2\/categories?post=10880"},{"taxonomy":"post_tag","embeddable":true,"href":"https:\/\/www.aihello.com\/resources\/wp-json\/wp\/v2\/tags?post=10880"},{"taxonomy":"republish_tag","embeddable":true,"href":"https:\/\/www.aihello.com\/resources\/wp-json\/wp\/v2\/republish_tag?post=10880"}],"curies":[{"name":"wp","href":"https:\/\/api.w.org\/{rel}","templated":true}]}}