query_engines

LlamaIndex Query Engines Guide

Complete guide to query engines, modes, and customization.

What are query engines?

Query engines power the retrieval and response generation in LlamaIndex: 1. Retrieve relevant chunks from index 2. Generate response using LLM + context 3. Return answer (optionally with sources)

Basic query engine

from llama_index.core import VectorStoreIndex

index = VectorStoreIndex.from_documents(documents)

# Default query engine
query_engine = index.as_query_engine()
response = query_engine.query("What is the main topic?")
print(response)

Response modes

1. Compact (default) - Best for most cases

query_engine = index.as_query_engine(
    response_mode="compact"
)

# Combines chunks that fit in context window
response = query_engine.query("Explain quantum computing")

2. Tree summarize - Hierarchical summarization

query_engine = index.as_query_engine(
    response_mode="tree_summarize"
)

# Builds summary tree from chunks
# Best for: Summarization tasks, many retrieved chunks
response = query_engine.query("Summarize all the key findings")

3. Simple summarize - Concatenate and summarize

query_engine = index.as_query_engine(
    response_mode="simple_summarize"
)

# Concatenates all chunks, then summarizes
# Fast but may lose context if too many chunks

4. Refine - Iterative refinement

query_engine = index.as_query_engine(
    response_mode="refine"
)

# Refines answer iteratively across chunks
# Most thorough, slowest
# Best for: Complex questions requiring synthesis

5. No text - Return nodes only

query_engine = index.as_query_engine(
    response_mode="no_text"
)

# Returns retrieved nodes without LLM response
# Useful for: Debugging retrieval, custom processing
response = query_engine.query("machine learning")
for node in response.source_nodes:
    print(node.text)

Configuration options

Similarity top-k

# Return top 3 most similar chunks
query_engine = index.as_query_engine(
    similarity_top_k=3  # Default: 2
)

Streaming

# Stream response tokens
query_engine = index.as_query_engine(streaming=True)

response = query_engine.query("Explain neural networks")
for text in response.response_gen:
    print(text, end="", flush=True)

Verbose mode

# Show retrieval and generation process
query_engine = index.as_query_engine(verbose=True)

response = query_engine.query("What is Python?")
# Prints: Retrieved chunks, prompts, LLM calls

Custom prompts

Text QA template

from llama_index.core import PromptTemplate

qa_prompt = PromptTemplate(
    "Context information is below.\n"
    "---------------------\n"
    "{context_str}\n"
    "---------------------\n"
    "Given the context, answer: {query_str}\n"
    "If the context doesn't contain the answer, say 'I don't know'.\n"
    "Answer: "
)

query_engine = index.as_query_engine(text_qa_template=qa_prompt)

Refine template

refine_prompt = PromptTemplate(
    "The original query is: {query_str}\n"
    "We have an existing answer: {existing_answer}\n"
    "We have new context: {context_msg}\n"
    "Refine the answer based on new context. "
    "If context isn't useful, return original answer.\n"
    "Refined Answer: "
)

query_engine = index.as_query_engine(
    response_mode="refine",
    refine_template=refine_prompt
)

Node postprocessors

Metadata filtering

from llama_index.core.postprocessor import MetadataReplacementPostProcessor

postprocessor = MetadataReplacementPostProcessor(
    target_metadata_key="window"  # Replace node content with window
)

query_engine = index.as_query_engine(
    node_postprocessors=[postprocessor]
)

Similarity cutoff

from llama_index.core.postprocessor import SimilarityPostprocessor

# Filter nodes below similarity threshold
postprocessor = SimilarityPostprocessor(similarity_cutoff=0.7)

query_engine = index.as_query_engine(
    node_postprocessors=[postprocessor]
)

Reranking

from llama_index.core.postprocessor import SentenceTransformerRerank

# Rerank retrieved nodes
reranker = SentenceTransformerRerank(
    model="cross-encoder/ms-marco-MiniLM-L-2-v2",
    top_n=3
)

query_engine = index.as_query_engine(
    node_postprocessors=[reranker],
    similarity_top_k=10  # Retrieve 10, rerank to 3
)

Advanced query engines

Sub-question query engine

from llama_index.core.query_engine import SubQuestionQueryEngine
from llama_index.core.tools import QueryEngineTool

# Multiple indices for different topics
python_index = VectorStoreIndex.from_documents(python_docs)
numpy_index = VectorStoreIndex.from_documents(numpy_docs)

# Create tools
python_tool = QueryEngineTool.from_defaults(
    query_engine=python_index.as_query_engine(),
    description="Useful for Python programming questions"
)
numpy_tool = QueryEngineTool.from_defaults(
    query_engine=numpy_index.as_query_engine(),
    description="Useful for NumPy array questions"
)

# Sub-question engine decomposes complex queries
query_engine = SubQuestionQueryEngine.from_defaults(
    query_engine_tools=[python_tool, numpy_tool]
)

# "How do I create numpy arrays in Python?" becomes:
# 1. Query numpy_tool about array creation
# 2. Query python_tool about syntax
# 3. Synthesize answers
response = query_engine.query("How do I create numpy arrays in Python?")

Router query engine

from llama_index.core.query_engine import RouterQueryEngine
from llama_index.core.selectors import LLMSingleSelector

# Route to appropriate index based on query
selector = LLMSingleSelector.from_defaults()

query_engine = RouterQueryEngine(
    selector=selector,
    query_engine_tools=[python_tool, numpy_tool]
)

# Automatically routes to correct index
response = query_engine.query("What is Python?")  # Routes to python_tool
response = query_engine.query("