title: Evaluate Production Conversations for RLHF
|
type: enterprise
|
group: Chat
|
order: 5
|
image: /static/templates/chat-rlhf.png
|
details: |
|
<h1>Bring production chats into this template to learn why your agent succeeds or fails</h1>
|
<dl>
|
<dt>Industry Applications</dt>
|
<dd>RLHF data collection, production conversation analysis, agent improvement, user preference learning, failure analysis, success pattern identification, fine-tuning data generation, quality monitoring, real-world behavior evaluation, feedback collection</dd>
|
<dt>Associated Models</dt>
|
<dd>RLHF training, preference learning, conversation evaluation, quality metrics</dd>
|
<dt>Domain Terminology</dt>
|
<dd>RLHF, human feedback, preference labels, conversation-level evaluation, per-message evaluation, production conversations, agent performance, success patterns</dd>
|
</dl>
|
config: |
|
<View>
|
<Style>
|
.chat {
|
border: 1px solid var(--color-neutral-border);
|
padding: var(--spacing-tight);
|
border-radius: var(--corner-radius-medium);
|
background-color: var(--color-neutral-background);
|
}
|
.evaluation {
|
border: 2px solid var(--color-accent-canteloupe-base);
|
background-color: var(--color-accent-canteloupe-subtlest);
|
color: var(--color-accent-canteloupe-bold);
|
padding: var(--spacing-tight);
|
border-radius: var(--corner-radius-medium);
|
margin-bottom: var(--spacing-base);
|
}
|
<!-- Choice text -->
|
.evaluation span {
|
color: var(--color-accent-canteloupe-bold);
|
}
|
<!-- Star rating -->
|
.evaluation .ant-rate-star.ant-rate-star-full span {
|
color: var(--color-accent-canteloupe-base);
|
}
|
|
.overall-chat {
|
border-bottom: 1px solid var(--color-neutral-border);
|
margin-bottom: var(--spacing-base);
|
}
|
.instructions {
|
color: var(--color-accent-canteloupe-bold);
|
background-color: var(--color-accent-canteloupe-subtlest);
|
padding: var(--spacing-tight);
|
border-radius: var(--corner-radius-medium);
|
border: 1px solid var(--color-accent-canteloupe-subtle);
|
}
|
<!-- Allow enlarging the instruction text -->
|
.lsf-richtext__container.lsf-htx-richtext {
|
font-size: 16px !important;
|
line-height: 1.6;
|
}
|
|
<!-- Remove excess height from the chat to allow space for instruction text -->
|
.htx-chat {
|
--excess-height: 275px;
|
background-color: var(--color-neutral-background);
|
}
|
</Style>
|
<View style="display: flex; gap: var(--spacing-wide);">
|
|
<!-- Left: conversation -->
|
<View className="chat" style="flex: 2;">
|
<View className="instructions">
|
<Text name="instructions" value="Review the conversation in detail.
|
As you read through it, click on individual messages to
|
provide feedback about accuracy, clarity, and intent." />
|
</View>
|
|
<Chat name="chat" value="$chat"
|
minMessages="2"
|
editable="false" />
|
</View>
|
|
<!-- Right: conversation and message evaluation -->
|
<View style="flex: 1; display: flex; flex-direction: column;" className="evaluation">
|
|
<!-- Evaluate the whole conversation -->
|
<View className="overall-chat">
|
<Header size="4">Overall quality of this conversation</Header>
|
<Rating name="rating" toName="chat" />
|
|
<View style="padding-top: var(--spacing-base);">
|
<Text name="add_comment" value="Add a comment (optional)" />
|
<TextArea name="conversation_comment" toName="chat" />
|
</View>
|
</View>
|
|
<!-- Per-message evaluation -->
|
<View style="flex: 1; overflow: auto;">
|
|
<!-- Only visible when no message is selected -->
|
<View visibleWhen="no-region-selected">
|
<Header value="Click on a message to evaluate" size="3"/>
|
</View>
|
|
<!-- Only visible when an assistant message is selected -->
|
<View visibleWhen="region-selected" whenRole="assistant">
|
<Header value="Response Accuracy"/>
|
<Rating name="assistant_response_accuracy" toName="chat" perRegion="true" maxRating="5" icon="star"/>
|
|
<Header value="Relevance"/>
|
<Choices name="relevance" toName="chat" perRegion="true" >
|
<Choice value="Highly relevant" />
|
<Choice value="Somewhat relevant" />
|
<Choice value="Not relevant" />
|
</Choices>
|
|
<Header value="Correctness"/>
|
<Choices name="correctness" toName="chat" perRegion="true" >
|
<Choice value="Accurate and factual" />
|
<Choice value="Mostly accurate with minor issues" />
|
<Choice value="Contains significant errors" />
|
</Choices>
|
|
<Header value="Tone"/>
|
<Choices name="tone" toName="chat" perRegion="true" choice="multiple" >
|
<Choice value="Too formal" />
|
<Choice value="Too casual" />
|
<Choice value="Appropriate" />
|
</Choices>
|
|
<Header value="Add a comment (optional)"/>
|
<TextArea perRegion="true" name="message_comment" toName="chat" />
|
</View>
|
</View>
|
</View>
|
</View>
|
</View>
|
|
<!-- {
|
"data": {
|
"text": "Review the conversation in detail. As you read through it, click on individual messages to provide feedback about accuracy, clarity, and intent.",
|
"chat": [
|
{
|
"role": "user",
|
"content": "I'm onboarding a new annotator. What are the first steps to ensure quality and speed?"
|
},
|
{
|
"role": "assistant",
|
"content": "Start by creating their account and assigning the Annotator role so permissions are scoped correctly. Add concise labeling instructions that appear before labeling. Use a small ground-truth set and require overlap (e.g., 2 annotators per task) with a reviewer to resolve disagreements. Enable comments so reviewers can leave precise, in-task feedback."
|
}
|
]
|
}
|
} -->
|
