curl --request POST \
--url https://softmax.com/api/observatory/v2/counterfactual-evals/{counterfactual_eval_id}/cancel \
--header 'Authorization: Bearer <token>'import requests
url = "https://softmax.com/api/observatory/v2/counterfactual-evals/{counterfactual_eval_id}/cancel"
headers = {"Authorization": "Bearer <token>"}
response = requests.post(url, headers=headers)
print(response.text)const options = {method: 'POST', headers: {Authorization: 'Bearer <token>'}};
fetch('https://softmax.com/api/observatory/v2/counterfactual-evals/{counterfactual_eval_id}/cancel', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://softmax.com/api/observatory/v2/counterfactual-evals/{counterfactual_eval_id}/cancel",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://softmax.com/api/observatory/v2/counterfactual-evals/{counterfactual_eval_id}/cancel"
req, _ := http.NewRequest("POST", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://softmax.com/api/observatory/v2/counterfactual-evals/{counterfactual_eval_id}/cancel")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://softmax.com/api/observatory/v2/counterfactual-evals/{counterfactual_eval_id}/cancel")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"id": "<string>",
"candidate_policy_version_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"baseline_policy_version_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"candidate_policy_label": "<string>",
"baseline_policy_label": "<string>",
"candidate_player_name": "<string>",
"baseline_player_name": "<string>",
"league_id": "<string>",
"league_name": "<string>",
"source": "tournament",
"n_requested": 123,
"n_paired": 123,
"status": "pending",
"skip_reason": "<string>",
"error": "<string>",
"verdict": "progression",
"net_score": 123,
"wins": 123,
"losses": 123,
"neutrals": 123,
"mean_score_delta": 123,
"experience_request_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"submission_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"league_policy_membership_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"idempotency_key": "<string>",
"created_by_user_id": "<string>",
"started_at": "2023-11-07T05:31:56Z",
"completed_at": "2023-11-07T05:31:56Z",
"created_at": "2023-11-07T05:31:56Z",
"pairs": [
{
"job_index": 123,
"baseline_episode_request_id": "<string>",
"candidate_episode_request_id": "<string>",
"candidate_episode_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"seed": 123,
"probe_slot": 123,
"opponent_policy_version_ids": [
"<string>"
],
"coworld_id": "<string>",
"variant_id": "<string>",
"baseline_score": 123,
"candidate_score": 123,
"score_delta": 123,
"outcome": 123
}
]
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Cancel counterfactual evaluation
Cancels pending evaluation work and its experience request.
curl --request POST \
--url https://softmax.com/api/observatory/v2/counterfactual-evals/{counterfactual_eval_id}/cancel \
--header 'Authorization: Bearer <token>'import requests
url = "https://softmax.com/api/observatory/v2/counterfactual-evals/{counterfactual_eval_id}/cancel"
headers = {"Authorization": "Bearer <token>"}
response = requests.post(url, headers=headers)
print(response.text)const options = {method: 'POST', headers: {Authorization: 'Bearer <token>'}};
fetch('https://softmax.com/api/observatory/v2/counterfactual-evals/{counterfactual_eval_id}/cancel', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://softmax.com/api/observatory/v2/counterfactual-evals/{counterfactual_eval_id}/cancel",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"net/http"
"io"
)
func main() {
url := "https://softmax.com/api/observatory/v2/counterfactual-evals/{counterfactual_eval_id}/cancel"
req, _ := http.NewRequest("POST", url, nil)
req.Header.Add("Authorization", "Bearer <token>")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://softmax.com/api/observatory/v2/counterfactual-evals/{counterfactual_eval_id}/cancel")
.header("Authorization", "Bearer <token>")
.asString();require 'uri'
require 'net/http'
url = URI("https://softmax.com/api/observatory/v2/counterfactual-evals/{counterfactual_eval_id}/cancel")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
response = http.request(request)
puts response.read_body{
"id": "<string>",
"candidate_policy_version_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"baseline_policy_version_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"candidate_policy_label": "<string>",
"baseline_policy_label": "<string>",
"candidate_player_name": "<string>",
"baseline_player_name": "<string>",
"league_id": "<string>",
"league_name": "<string>",
"source": "tournament",
"n_requested": 123,
"n_paired": 123,
"status": "pending",
"skip_reason": "<string>",
"error": "<string>",
"verdict": "progression",
"net_score": 123,
"wins": 123,
"losses": 123,
"neutrals": 123,
"mean_score_delta": 123,
"experience_request_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"submission_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"league_policy_membership_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"idempotency_key": "<string>",
"created_by_user_id": "<string>",
"started_at": "2023-11-07T05:31:56Z",
"completed_at": "2023-11-07T05:31:56Z",
"created_at": "2023-11-07T05:31:56Z",
"pairs": [
{
"job_index": 123,
"baseline_episode_request_id": "<string>",
"candidate_episode_request_id": "<string>",
"candidate_episode_id": "3c90c3cc-0d44-4b50-8888-8dd25736052a",
"seed": 123,
"probe_slot": 123,
"opponent_policy_version_ids": [
"<string>"
],
"coworld_id": "<string>",
"variant_id": "<string>",
"baseline_score": 123,
"candidate_score": 123,
"score_delta": 123,
"outcome": 123
}
]
}{
"detail": [
{
"loc": [
"<string>"
],
"msg": "<string>",
"type": "<string>",
"input": "<unknown>",
"ctx": {}
}
]
}Authorizations
Bearer token issued by Softmax
Path Parameters
Counterfactual evaluation to cancel.
^cfeval_[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$Response
Successful Response
Counterfactual evaluation ID.
Candidate policy version being evaluated.
Baseline policy version used for comparison.
Display label for the candidate policy version.
Display label for the baseline policy version.
Player attributed to the candidate.
Player attributed to the baseline.
League that supplied the evaluation settings.
League name.
Workflow that created the evaluation.
tournament, experience_request Number of paired comparisons requested.
Number of paired comparisons successfully scored.
Current evaluation state.
pending, running, completed, failed, skipped, cancelled Reason the evaluation was skipped.
Failure message, or null when no failure was recorded.
Final comparison verdict.
progression, regression, neutral Win-loss margin divided by paired comparisons, when scoring is complete.
Number of comparisons won by the candidate.
Number of comparisons lost by the candidate.
Number of comparisons with a neutral outcome.
Mean candidate score change, when scoring is complete.
Experience Request created to run the comparisons.
League submission that triggered the evaluation.
League membership associated with the evaluation.
Key that makes repeated create requests return the same evaluation.
User that created the evaluation.
Time when evaluation work started.
Time when the evaluation reached a final state.
Time when the evaluation was created.
Paired baseline and candidate episodes in this evaluation.
Show child attributes
Show child attributes