Create Custom Metrics
curl --request POST \
--url https://api.getbluejay.ai/v1/create-custom-metrics \
--header 'Content-Type: application/json' \
--header 'X-API-Key: <x-api-key>' \
--data '
{
"metrics": [
{
"name": "<string>",
"bluejay_as_code_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"agent_id": 123,
"agent_ids": [
123
],
"prompt": "<string>",
"summary": "<string>",
"metric_type": "llm_judge",
"settings": {},
"eval_segment": "all",
"scope": "all",
"min_value": 123,
"max_value": 123,
"category": "<string>",
"tags": [
"<string>"
],
"scoring_guidance": "<string>",
"eval_modality": "AUTO",
"model": "<string>",
"audio_model": "<string>",
"temperature": 123,
"enum_options": [
"<string>"
],
"tool_names": [
"<string>"
],
"json_schema": {},
"allow_not_applicable": false,
"template_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a"
}
]
}
'import requests
url = "https://api.getbluejay.ai/v1/create-custom-metrics"
payload = { "metrics": [
{
"name": "<string>",
"bluejay_as_code_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"agent_id": 123,
"agent_ids": [123],
"prompt": "<string>",
"summary": "<string>",
"metric_type": "llm_judge",
"settings": {},
"eval_segment": "all",
"scope": "all",
"min_value": 123,
"max_value": 123,
"category": "<string>",
"tags": ["<string>"],
"scoring_guidance": "<string>",
"eval_modality": "AUTO",
"model": "<string>",
"audio_model": "<string>",
"temperature": 123,
"enum_options": ["<string>"],
"tool_names": ["<string>"],
"json_schema": {},
"allow_not_applicable": False,
"template_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a"
}
] }
headers = {
"X-API-Key": "<x-api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'X-API-Key': '<x-api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
metrics: [
{
name: '<string>',
bluejay_as_code_id: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
agent_id: 123,
agent_ids: [123],
prompt: '<string>',
summary: '<string>',
metric_type: 'llm_judge',
settings: {},
eval_segment: 'all',
scope: 'all',
min_value: 123,
max_value: 123,
category: '<string>',
tags: ['<string>'],
scoring_guidance: '<string>',
eval_modality: 'AUTO',
model: '<string>',
audio_model: '<string>',
temperature: 123,
enum_options: ['<string>'],
tool_names: ['<string>'],
json_schema: {},
allow_not_applicable: false,
template_id: '3c90c3cc-0d44-4b50-8888-8dd25736052a'
}
]
})
};
fetch('https://api.getbluejay.ai/v1/create-custom-metrics', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.getbluejay.ai/v1/create-custom-metrics",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'metrics' => [
[
'name' => '<string>',
'bluejay_as_code_id' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'agent_id' => 123,
'agent_ids' => [
123
],
'prompt' => '<string>',
'summary' => '<string>',
'metric_type' => 'llm_judge',
'settings' => [
],
'eval_segment' => 'all',
'scope' => 'all',
'min_value' => 123,
'max_value' => 123,
'category' => '<string>',
'tags' => [
'<string>'
],
'scoring_guidance' => '<string>',
'eval_modality' => 'AUTO',
'model' => '<string>',
'audio_model' => '<string>',
'temperature' => 123,
'enum_options' => [
'<string>'
],
'tool_names' => [
'<string>'
],
'json_schema' => [
],
'allow_not_applicable' => false,
'template_id' => '3c90c3cc-0d44-4b50-8888-8dd25736052a'
]
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"X-API-Key: <x-api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.getbluejay.ai/v1/create-custom-metrics"
payload := strings.NewReader("{\n \"metrics\": [\n {\n \"name\": \"<string>\",\n \"bluejay_as_code_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"agent_id\": 123,\n \"agent_ids\": [\n 123\n ],\n \"prompt\": \"<string>\",\n \"summary\": \"<string>\",\n \"metric_type\": \"llm_judge\",\n \"settings\": {},\n \"eval_segment\": \"all\",\n \"scope\": \"all\",\n \"min_value\": 123,\n \"max_value\": 123,\n \"category\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"scoring_guidance\": \"<string>\",\n \"eval_modality\": \"AUTO\",\n \"model\": \"<string>\",\n \"audio_model\": \"<string>\",\n \"temperature\": 123,\n \"enum_options\": [\n \"<string>\"\n ],\n \"tool_names\": [\n \"<string>\"\n ],\n \"json_schema\": {},\n \"allow_not_applicable\": false,\n \"template_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("X-API-Key", "<x-api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.getbluejay.ai/v1/create-custom-metrics")
.header("X-API-Key", "<x-api-key>")
.header("Content-Type", "application/json")
.body("{\n \"metrics\": [\n {\n \"name\": \"<string>\",\n \"bluejay_as_code_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"agent_id\": 123,\n \"agent_ids\": [\n 123\n ],\n \"prompt\": \"<string>\",\n \"summary\": \"<string>\",\n \"metric_type\": \"llm_judge\",\n \"settings\": {},\n \"eval_segment\": \"all\",\n \"scope\": \"all\",\n \"min_value\": 123,\n \"max_value\": 123,\n \"category\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"scoring_guidance\": \"<string>\",\n \"eval_modality\": \"AUTO\",\n \"model\": \"<string>\",\n \"audio_model\": \"<string>\",\n \"temperature\": 123,\n \"enum_options\": [\n \"<string>\"\n ],\n \"tool_names\": [\n \"<string>\"\n ],\n \"json_schema\": {},\n \"allow_not_applicable\": false,\n \"template_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.getbluejay.ai/v1/create-custom-metrics")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["X-API-Key"] = '<x-api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"metrics\": [\n {\n \"name\": \"<string>\",\n \"bluejay_as_code_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"agent_id\": 123,\n \"agent_ids\": [\n 123\n ],\n \"prompt\": \"<string>\",\n \"summary\": \"<string>\",\n \"metric_type\": \"llm_judge\",\n \"settings\": {},\n \"eval_segment\": \"all\",\n \"scope\": \"all\",\n \"min_value\": 123,\n \"max_value\": 123,\n \"category\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"scoring_guidance\": \"<string>\",\n \"eval_modality\": \"AUTO\",\n \"model\": \"<string>\",\n \"audio_model\": \"<string>\",\n \"temperature\": 123,\n \"enum_options\": [\n \"<string>\"\n ],\n \"tool_names\": [\n \"<string>\"\n ],\n \"json_schema\": {},\n \"allow_not_applicable\": false,\n \"template_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"metrics": [
{
"id": "<string>",
"name": "<string>",
"response_type": "pass_fail",
"created_at": "2023-11-07T05:31:56Z",
"bluejay_as_code_id": "<string>",
"prompt": "<string>",
"description": "<string>",
"summary": "<string>",
"metric_type": "llm_judge",
"eval_method": "llm_judge",
"settings": {},
"eval_segment": "all",
"scope": "all",
"agent_ids": [
123
],
"min_value": 123,
"max_value": 123,
"category": "<string>",
"tags": [
"<string>"
],
"scoring_guidance": "<string>",
"updated_at": "2023-11-07T05:31:56Z",
"updated_by": "<string>",
"created_by": "<string>",
"eval_modality": "AUDIO",
"model": "<string>",
"audio_model": "<string>",
"temperature": 123,
"enum_options": [
"<string>"
],
"tool_names": [
"<string>"
],
"json_schema": {},
"allow_not_applicable": false,
"eval_modality_auto_selected": false,
"template_id": "<string>"
}
]
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Custom Metrics
Create Custom Metrics
deprecated
Create multiple custom metrics synchronously using a single bulk insert.
POST
/
v1
/
create-custom-metrics
Create Custom Metrics
curl --request POST \
--url https://api.getbluejay.ai/v1/create-custom-metrics \
--header 'Content-Type: application/json' \
--header 'X-API-Key: <x-api-key>' \
--data '
{
"metrics": [
{
"name": "<string>",
"bluejay_as_code_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"agent_id": 123,
"agent_ids": [
123
],
"prompt": "<string>",
"summary": "<string>",
"metric_type": "llm_judge",
"settings": {},
"eval_segment": "all",
"scope": "all",
"min_value": 123,
"max_value": 123,
"category": "<string>",
"tags": [
"<string>"
],
"scoring_guidance": "<string>",
"eval_modality": "AUTO",
"model": "<string>",
"audio_model": "<string>",
"temperature": 123,
"enum_options": [
"<string>"
],
"tool_names": [
"<string>"
],
"json_schema": {},
"allow_not_applicable": false,
"template_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a"
}
]
}
'import requests
url = "https://api.getbluejay.ai/v1/create-custom-metrics"
payload = { "metrics": [
{
"name": "<string>",
"bluejay_as_code_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"agent_id": 123,
"agent_ids": [123],
"prompt": "<string>",
"summary": "<string>",
"metric_type": "llm_judge",
"settings": {},
"eval_segment": "all",
"scope": "all",
"min_value": 123,
"max_value": 123,
"category": "<string>",
"tags": ["<string>"],
"scoring_guidance": "<string>",
"eval_modality": "AUTO",
"model": "<string>",
"audio_model": "<string>",
"temperature": 123,
"enum_options": ["<string>"],
"tool_names": ["<string>"],
"json_schema": {},
"allow_not_applicable": False,
"template_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a"
}
] }
headers = {
"X-API-Key": "<x-api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {'X-API-Key': '<x-api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
metrics: [
{
name: '<string>',
bluejay_as_code_id: '3c90c3cc-0d44-4b50-8888-8dd25736052a',
agent_id: 123,
agent_ids: [123],
prompt: '<string>',
summary: '<string>',
metric_type: 'llm_judge',
settings: {},
eval_segment: 'all',
scope: 'all',
min_value: 123,
max_value: 123,
category: '<string>',
tags: ['<string>'],
scoring_guidance: '<string>',
eval_modality: 'AUTO',
model: '<string>',
audio_model: '<string>',
temperature: 123,
enum_options: ['<string>'],
tool_names: ['<string>'],
json_schema: {},
allow_not_applicable: false,
template_id: '3c90c3cc-0d44-4b50-8888-8dd25736052a'
}
]
})
};
fetch('https://api.getbluejay.ai/v1/create-custom-metrics', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.getbluejay.ai/v1/create-custom-metrics",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'metrics' => [
[
'name' => '<string>',
'bluejay_as_code_id' => '3c90c3cc-0d44-4b50-8888-8dd25736052a',
'agent_id' => 123,
'agent_ids' => [
123
],
'prompt' => '<string>',
'summary' => '<string>',
'metric_type' => 'llm_judge',
'settings' => [
],
'eval_segment' => 'all',
'scope' => 'all',
'min_value' => 123,
'max_value' => 123,
'category' => '<string>',
'tags' => [
'<string>'
],
'scoring_guidance' => '<string>',
'eval_modality' => 'AUTO',
'model' => '<string>',
'audio_model' => '<string>',
'temperature' => 123,
'enum_options' => [
'<string>'
],
'tool_names' => [
'<string>'
],
'json_schema' => [
],
'allow_not_applicable' => false,
'template_id' => '3c90c3cc-0d44-4b50-8888-8dd25736052a'
]
]
]),
CURLOPT_HTTPHEADER => [
"Content-Type: application/json",
"X-API-Key: <x-api-key>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.getbluejay.ai/v1/create-custom-metrics"
payload := strings.NewReader("{\n \"metrics\": [\n {\n \"name\": \"<string>\",\n \"bluejay_as_code_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"agent_id\": 123,\n \"agent_ids\": [\n 123\n ],\n \"prompt\": \"<string>\",\n \"summary\": \"<string>\",\n \"metric_type\": \"llm_judge\",\n \"settings\": {},\n \"eval_segment\": \"all\",\n \"scope\": \"all\",\n \"min_value\": 123,\n \"max_value\": 123,\n \"category\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"scoring_guidance\": \"<string>\",\n \"eval_modality\": \"AUTO\",\n \"model\": \"<string>\",\n \"audio_model\": \"<string>\",\n \"temperature\": 123,\n \"enum_options\": [\n \"<string>\"\n ],\n \"tool_names\": [\n \"<string>\"\n ],\n \"json_schema\": {},\n \"allow_not_applicable\": false,\n \"template_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("X-API-Key", "<x-api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.getbluejay.ai/v1/create-custom-metrics")
.header("X-API-Key", "<x-api-key>")
.header("Content-Type", "application/json")
.body("{\n \"metrics\": [\n {\n \"name\": \"<string>\",\n \"bluejay_as_code_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"agent_id\": 123,\n \"agent_ids\": [\n 123\n ],\n \"prompt\": \"<string>\",\n \"summary\": \"<string>\",\n \"metric_type\": \"llm_judge\",\n \"settings\": {},\n \"eval_segment\": \"all\",\n \"scope\": \"all\",\n \"min_value\": 123,\n \"max_value\": 123,\n \"category\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"scoring_guidance\": \"<string>\",\n \"eval_modality\": \"AUTO\",\n \"model\": \"<string>\",\n \"audio_model\": \"<string>\",\n \"temperature\": 123,\n \"enum_options\": [\n \"<string>\"\n ],\n \"tool_names\": [\n \"<string>\"\n ],\n \"json_schema\": {},\n \"allow_not_applicable\": false,\n \"template_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.getbluejay.ai/v1/create-custom-metrics")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["X-API-Key"] = '<x-api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"metrics\": [\n {\n \"name\": \"<string>\",\n \"bluejay_as_code_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\",\n \"agent_id\": 123,\n \"agent_ids\": [\n 123\n ],\n \"prompt\": \"<string>\",\n \"summary\": \"<string>\",\n \"metric_type\": \"llm_judge\",\n \"settings\": {},\n \"eval_segment\": \"all\",\n \"scope\": \"all\",\n \"min_value\": 123,\n \"max_value\": 123,\n \"category\": \"<string>\",\n \"tags\": [\n \"<string>\"\n ],\n \"scoring_guidance\": \"<string>\",\n \"eval_modality\": \"AUTO\",\n \"model\": \"<string>\",\n \"audio_model\": \"<string>\",\n \"temperature\": 123,\n \"enum_options\": [\n \"<string>\"\n ],\n \"tool_names\": [\n \"<string>\"\n ],\n \"json_schema\": {},\n \"allow_not_applicable\": false,\n \"template_id\": \"3c90c3cc-0d44-4b50-8888-8dd25736052a\"\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"metrics": [
{
"id": "<string>",
"name": "<string>",
"response_type": "pass_fail",
"created_at": "2023-11-07T05:31:56Z",
"bluejay_as_code_id": "<string>",
"prompt": "<string>",
"description": "<string>",
"summary": "<string>",
"metric_type": "llm_judge",
"eval_method": "llm_judge",
"settings": {},
"eval_segment": "all",
"scope": "all",
"agent_ids": [
123
],
"min_value": 123,
"max_value": 123,
"category": "<string>",
"tags": [
"<string>"
],
"scoring_guidance": "<string>",
"updated_at": "2023-11-07T05:31:56Z",
"updated_by": "<string>",
"created_by": "<string>",
"eval_modality": "AUDIO",
"model": "<string>",
"audio_model": "<string>",
"temperature": 123,
"enum_options": [
"<string>"
],
"tool_names": [
"<string>"
],
"json_schema": {},
"allow_not_applicable": false,
"eval_modality_auto_selected": false,
"template_id": "<string>"
}
]
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Integration Prompt for AI Agents
# Bluejay — Testing & Monitoring Platform for Conversational AI Agents
You are a senior backend engineer integrating the Bluejay API. Think step-by-step: first understand the endpoint, then plan the integration, then implement with minimal changes.
## Create Custom Metrics — POST /v1/create-custom-metrics
> **What this endpoint does:** Create multiple custom metrics synchronously using a single bulk insert.
**Endpoint:** POST `https://api.getbluejay.ai/v1/create-custom-metrics`
**Auth:** `X-API-Key` header
**Content-Type:** application/json
### Required Parameters
| Name | Type | Description |
|------|------|-------------|
| X-API-Key | string | API key required to authenticate requests. |
| metrics | array[object] | List of custom metrics to create |
### Request Body (required fields)
```json
{
"metrics": [
{
"agent_id": 123,
"agent_ids": [
123
],
"name": "example_name",
"description": "string",
"response_type": "pass_fail",
"min_value": 1.0,
"max_value": 1.0,
"category": "string",
"tags": [
"string"
],
"scoring_guidance": "string",
"eval_route": "AUDIO",
"model": "string",
"temperature": 1.0,
"enum_options": [
"string"
],
"allow_not_applicable": false
}
]
}
```
Refer to the full schema at https://docs.getbluejay.ai/api-reference/endpoint/create-custom-metrics. Include optional fields that serve the goal of setting up for testing and monitoring on Bluejay.
### Example
**POST with body:**
```python
import requests
def create_custom_metrics(payload: dict, api_key: str) -> dict:
url = "https://api.getbluejay.ai/v1/create-custom-metrics"
headers = {"X-API-Key": api_key}
response = requests.post(url, headers=headers, json=payload)
response.raise_for_status()
return response.json()
```
### Constraints
- Minimal changes — only add/change files needed for this integration.
- Match existing codebase patterns (naming, file structure, error handling).
- Include error handling for 422: Validation Error.
### Integration Checklist
Before writing code, verify:
1. Which module/service owns this API domain in the codebase?
2. What HTTP client and error-handling patterns does the project use?
3. Are there existing types/interfaces to extend?
Then implement the integration, export it, and confirm it compiles/passes lint.
Body
application/json
Request model for creating multiple custom metrics
List of custom metrics to create
Show child attributes
Show child attributes
Response
Successful Response
Response model for bulk creating custom metrics
List of created custom metrics
Show child attributes
Show child attributes