---
|
test_name: tasks-all-fields-postgre
|
strict: false
|
marks:
|
- usefixtures:
|
- django_live_url
|
- skipif: "'default' not in '{tavern.env_vars.DJANGO_DB}'"
|
stages:
|
|
- id: signup
|
type: ref
|
|
- id: create_project
|
name: create_project
|
request:
|
data:
|
title: Test Draft 1
|
show_collab_predictions: true
|
label_config: '<View><Text name="text" value="$text"/><Choices name="text_class" toName="text"><Choice value="class_A"/><Choice value="class_B"/><Choice value="class_TESTING"/><Choice value="class_PREDICTIONS_TESTING"/></Choices></View>'
|
method: POST
|
url: '{django_live_url}/api/projects'
|
response:
|
save:
|
json:
|
project_pk: id
|
created_by: created_by.id
|
status_code: 201
|
|
- name: create_filter_tasks
|
request:
|
files:
|
json_file: tests/data_manager/tasks_annotations_predictions.json
|
headers:
|
content-type: multipart/form-data
|
method: POST
|
url: '{django_live_url}/api/projects/{project_pk}/import'
|
response:
|
json:
|
annotation_count: 4
|
prediction_count: 3
|
task_count: 6
|
status_code: 201
|
|
- name: setup_views
|
request:
|
method: POST
|
url: '{django_live_url}/api/dm/views/?project={project_pk}'
|
json: {
|
"data": {
|
"filters": {
|
"conjunction": "or",
|
"items": [
|
{
|
"filter": "filter:tasks:data.text",
|
"operator": "equal",
|
"type": "String",
|
"value": "Test example phrase 1"
|
},
|
{
|
"filter": "filter:tasks:data.text",
|
"operator": "contains",
|
"type": "String",
|
"value": "x"
|
},
|
{
|
"filter": "filter:tasks:annotations_results",
|
"operator": "contains",
|
"type": "String",
|
"value": "TESTING"
|
},
|
{
|
"filter": "filter:tasks:predictions_results",
|
"operator": "contains",
|
"type": "String",
|
"value": "PREDICTIONS"
|
}
|
]
|
},
|
"ordering": [
|
"tasks:data.text"
|
]
|
},
|
"project": !int "{project_pk}"
|
}
|
response:
|
save:
|
json:
|
view: "@"
|
status_code: 201
|
|
- name: get_tasks
|
request:
|
method: GET
|
url: '{django_live_url}/api/tasks?fields=all&view={view.id}'
|
response:
|
status_code: 200
|
verify_response_with:
|
function: label_studio.tests.utils:save_response
|
json: {
|
"total_annotations": 4,
|
"total_predictions": 3,
|
"total": 4,
|
"tasks": [
|
{
|
"cancelled_annotations": 0,
|
"storage_filename": null,
|
"annotations_results": "",
|
"data": {
|
"text": "opop",
|
"int_field": 42
|
},
|
|
"predictions_results": !raw "[{type:choices, value:{choices:[class_PREDICTIONS_TESTING]}, to_name:text, from_name:text_class}]",
|
"predictions_score": null,
|
"total_annotations": 0,
|
"total_predictions": 1,
|
"annotations_ids": "",
|
"annotations": [],
|
"predictions": [
|
{
|
"model_version": "undefined",
|
"created_ago": "0\u00a0minutes",
|
"result": [
|
{
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_PREDICTIONS_TESTING"
|
]
|
},
|
"to_name": "text",
|
"from_name": "text_class"
|
}
|
],
|
"score": null,
|
"cluster": null,
|
"neighbors": null,
|
"mislabeling": 0.0
|
}
|
],
|
"drafts": [],
|
"annotators": [],
|
"avg_lead_time": null
|
|
},
|
{
|
"cancelled_annotations": 0,
|
"storage_filename": null,
|
"annotations_results": !raw "[{type:choices, value:{choices:[class_A]}, to_name:text, from_name:text_class}]",
|
"data": {
|
"text": "Test example phrase 1",
|
"int_field": 1
|
},
|
|
"predictions_results": !raw "[{type:choices, value:{choices:[class_A]}, to_name:text, from_name:text_class}]",
|
"predictions_score": null,
|
"total_annotations": 1,
|
"total_predictions": 1,
|
"annotations_ids": !anystr "",
|
"annotations": [
|
{
|
|
"created_username": " test_suites_user@heartex.com, {created_by}",
|
"created_ago": "0\u00a0minutes",
|
"completed_by": !int "{created_by}",
|
"result": [
|
{
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_A"
|
]
|
},
|
"to_name": "text",
|
"from_name": "text_class"
|
}
|
],
|
"was_cancelled": false,
|
"ground_truth": true,
|
"lead_time": 3.0
|
}
|
],
|
"predictions": [
|
{
|
"model_version": "model_version_A",
|
"created_ago": "0\u00a0minutes",
|
"result": [
|
{
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_A"
|
]
|
},
|
"to_name": "text",
|
"from_name": "text_class"
|
}
|
],
|
"score": null,
|
"cluster": null,
|
"neighbors": null,
|
"mislabeling": 0.0
|
}
|
],
|
"drafts": [],
|
"annotators": [
|
!int "{created_by}"
|
],
|
"avg_lead_time": 3.0
|
},
|
{
|
"cancelled_annotations": 0,
|
"storage_filename": null,
|
"annotations_results": !anystr "",
|
"data": {
|
"text": "x2",
|
"int_field": 20
|
},
|
"predictions_results": !anystr "",
|
"predictions_score": null,
|
"total_annotations": 2,
|
"total_predictions": 1,
|
"annotations_ids": !anystr "",
|
"annotations": [
|
{
|
"created_username": " test_suites_user@heartex.com, {created_by}",
|
"created_ago": "0\u00a0minutes",
|
"result": [
|
{
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_A"
|
]
|
},
|
"to_name": "text",
|
"from_name": "text_class"
|
}
|
],
|
"was_cancelled": false,
|
"ground_truth": true,
|
"lead_time": 3.0
|
},
|
{
|
"created_username": " test_suites_user@heartex.com, {created_by}",
|
"created_ago": "0\u00a0minutes",
|
"result": [
|
{
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_A"
|
]
|
},
|
"to_name": "text",
|
"from_name": "text_class"
|
}
|
],
|
"was_cancelled": false,
|
"ground_truth": true,
|
"lead_time": 5.0
|
}
|
],
|
"predictions": [
|
{
|
|
"model_version": "model_version_A",
|
"created_ago": "0\u00a0minutes",
|
"result": [
|
{
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_A"
|
]
|
},
|
"to_name": "text",
|
"from_name": "text_class"
|
}
|
],
|
"score": null,
|
"cluster": null,
|
"neighbors": null,
|
"mislabeling": 0.0
|
}
|
],
|
"drafts": [],
|
"annotators": [
|
!int "{created_by}"
|
],
|
"avg_lead_time": 4.0
|
},
|
{
|
"cancelled_annotations": 0,
|
"storage_filename": null,
|
|
"annotations_results": !raw "[{type:choices, value:{choices:[class_TESTING]}, to_name:text, from_name:text_class}]",
|
"data": {
|
"text": "yoyo",
|
"int_field": "99"
|
},
|
|
"predictions_results": "",
|
"predictions_score": null,
|
"total_annotations": 1,
|
"total_predictions": 0,
|
"annotations_ids": !anystr "",
|
"annotations": [
|
{
|
"created_ago": "0\u00a0minutes",
|
"completed_by": !int "{created_by}",
|
"result": [
|
{
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_TESTING"
|
]
|
},
|
"to_name": "text",
|
"from_name": "text_class"
|
}
|
],
|
"was_cancelled": false,
|
"ground_truth": true,
|
"lead_time": null
|
|
}
|
],
|
"predictions": [],
|
"drafts": [],
|
|
"annotators": [
|
!int "{created_by}"
|
]
|
|
}
|
]
|
}
|
|
---
|
test_name: tasks-all-fields-sqlite
|
strict: false
|
marks:
|
- skipif: "'default' in '{tavern.env_vars.DJANGO_DB}'"
|
- usefixtures:
|
- django_live_url
|
stages:
|
|
- id: signup
|
type: ref
|
|
- id: create_project
|
name: create_project
|
request:
|
data:
|
title: Test Draft 1
|
show_collab_predictions: true
|
label_config: '<View><Text name="text" value="$text"/><Choices name="text_class" toName="text"><Choice value="class_A"/><Choice value="class_B"/><Choice value="class_TESTING"/><Choice value="class_PREDICTIONS_TESTING"/></Choices></View>'
|
method: POST
|
url: '{django_live_url}/api/projects'
|
response:
|
save:
|
json:
|
project_pk: id
|
created_by: created_by.id
|
status_code: 201
|
|
- name: create_filter_tasks
|
request:
|
files:
|
json_file: tests/data_manager/tasks_annotations_predictions.json
|
headers:
|
content-type: multipart/form-data
|
method: POST
|
url: '{django_live_url}/api/projects/{project_pk}/import'
|
response:
|
json:
|
annotation_count: 4
|
prediction_count: 3
|
task_count: 6
|
status_code: 201
|
|
- name: setup_views
|
request:
|
method: POST
|
url: '{django_live_url}/api/dm/views/?project={project_pk}'
|
json: {
|
"data": {
|
"filters": {
|
"conjunction": "or",
|
"items": [
|
{
|
"filter": "filter:tasks:data.text",
|
"operator": "equal",
|
"type": "String",
|
"value": "Test example phrase 1"
|
},
|
{
|
"filter": "filter:tasks:data.text",
|
"operator": "contains",
|
"type": "String",
|
"value": "x"
|
},
|
{
|
"filter": "filter:tasks:annotations_results",
|
"operator": "contains",
|
"type": "String",
|
"value": "TESTING"
|
},
|
{
|
"filter": "filter:tasks:predictions_results",
|
"operator": "contains",
|
"type": "String",
|
"value": "PREDICTIONS"
|
}
|
]
|
},
|
"ordering": [
|
"tasks:data.text"
|
]
|
},
|
"project": !int "{project_pk}"
|
}
|
response:
|
save:
|
json:
|
view: "@"
|
status_code: 201
|
|
- name: get_tasks
|
request:
|
method: GET
|
url: '{django_live_url}/api/tasks?fields=all&view={view.id}'
|
response:
|
status_code: 200
|
# verify_response_with:
|
# function: label_studio.tests.utils:save_response
|
json:
|
{
|
"total_annotations": 4,
|
"total_predictions": 3,
|
"total": 4,
|
"tasks": [
|
{
|
"cancelled_annotations": 0,
|
"storage_filename": null,
|
"annotations_results": !raw "[{from_name: text_class, to_name: text, type: choices, value: {choices: [class_A]}}]",
|
"data": {
|
"text": "Test example phrase 1",
|
"int_field": 1
|
},
|
"predictions_results": !raw "[{from_name: text_class, to_name: text, type: choices, value: {choices: [class_A]}}]",
|
"predictions_score": null,
|
"total_annotations": 1,
|
"total_predictions": 1,
|
"annotations_ids": !anystr "",
|
"annotations": [
|
{
|
|
"created_username": " test_suites_user@heartex.com, {created_by}",
|
"created_ago": "0\u00a0minutes",
|
"completed_by": !int "{created_by}",
|
"result": [
|
{
|
"from_name": "text_class",
|
"to_name": "text",
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_A"
|
]
|
}
|
}
|
],
|
"was_cancelled": false,
|
"ground_truth": true,
|
|
"lead_time": 3.0
|
|
}
|
],
|
"predictions": [
|
{
|
|
"model_version": "model_version_A",
|
"created_ago": "0\u00a0minutes",
|
"result": [
|
{
|
"from_name": "text_class",
|
"to_name": "text",
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_A"
|
]
|
}
|
}
|
],
|
"score": null,
|
"cluster": null,
|
"neighbors": null,
|
"mislabeling": 0.0,
|
|
|
}
|
],
|
"drafts": [],
|
"annotators": [
|
!int "{created_by}"
|
],
|
"project": !int "{project_pk}"
|
},
|
{
|
"cancelled_annotations": 0,
|
"storage_filename": null,
|
"annotations_results": "",
|
"data": {
|
"int_field": 42,
|
"text": "opop"
|
},
|
"predictions_results": !raw "[{from_name: text_class, to_name: text, type: choices, value: {choices: [class_PREDICTIONS_TESTING]}}]",
|
"predictions_score": null,
|
"total_annotations": 0,
|
"total_predictions": 1,
|
"annotations_ids": !anystr "",
|
"annotations": [],
|
"predictions": [
|
{
|
|
"model_version": "undefined",
|
"created_ago": "0\u00a0minutes",
|
"result": [
|
{
|
"from_name": "text_class",
|
"to_name": "text",
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_PREDICTIONS_TESTING"
|
]
|
}
|
}
|
],
|
"score": null,
|
"cluster": null,
|
"neighbors": null,
|
"mislabeling": 0.0,
|
|
|
}
|
],
|
"drafts": [],
|
"annotators": [],
|
"project": !int "{project_pk}"
|
},
|
{
|
"cancelled_annotations": 0,
|
"storage_filename": null,
|
"annotations_results": !anystr "",
|
"data": {
|
"int_field": 20,
|
"text": "x2"
|
},
|
"predictions_results": !anystr "",
|
"predictions_score": null,
|
"total_annotations": 2,
|
"total_predictions": 1,
|
"annotations_ids": !anystr "",
|
"annotations": [
|
{
|
|
"created_username": " test_suites_user@heartex.com, {created_by}",
|
"created_ago": "0\u00a0minutes",
|
"completed_by": !int "{created_by}",
|
"result": [
|
{
|
"from_name": "text_class",
|
"to_name": "text",
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_A"
|
]
|
}
|
}
|
],
|
"was_cancelled": false,
|
"ground_truth": true,
|
|
"lead_time": 3.0
|
|
},
|
{
|
|
"created_username": " test_suites_user@heartex.com, {created_by}",
|
"created_ago": "0\u00a0minutes",
|
"completed_by": !int "{created_by}",
|
"result": [
|
{
|
"from_name": "text_class",
|
"to_name": "text",
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_A"
|
]
|
}
|
}
|
],
|
"was_cancelled": false,
|
"ground_truth": true,
|
|
"lead_time": 5.0
|
|
}
|
],
|
"predictions": [
|
{
|
|
"model_version": "model_version_A",
|
"created_ago": "0\u00a0minutes",
|
"result": [
|
{
|
"from_name": "text_class",
|
"to_name": "text",
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_A"
|
]
|
}
|
}
|
],
|
"score": null,
|
"cluster": null,
|
"neighbors": null,
|
"mislabeling": 0.0,
|
|
|
}
|
],
|
"drafts": [],
|
"annotators": [
|
!int "{created_by}"
|
],
|
"project": !int "{project_pk}"
|
},
|
{
|
"cancelled_annotations": 0,
|
"storage_filename": null,
|
"annotations_results": !raw "[{from_name: text_class, to_name: text, type: choices, value: {choices: [class_TESTING]}}]",
|
"data": {
|
"int_field": "99",
|
"text": "yoyo"
|
},
|
"predictions_results": "",
|
"predictions_score": null,
|
"total_annotations": 1,
|
"total_predictions": 0,
|
"annotations_ids": !anystr "",
|
"annotations": [
|
{
|
|
"created_username": " test_suites_user@heartex.com, {created_by}",
|
"created_ago": "0\u00a0minutes",
|
"completed_by": !int "{created_by}",
|
"result": [
|
{
|
"from_name": "text_class",
|
"to_name": "text",
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_TESTING"
|
]
|
}
|
}
|
],
|
"was_cancelled": false,
|
"ground_truth": true,
|
|
"lead_time": null
|
|
}
|
],
|
"predictions": [],
|
"drafts": [],
|
"annotators": [
|
!int "{created_by}"
|
],
|
"project": !int "{project_pk}"
|
}
|
]
|
}
|
|
---
|
test_name: tasks-annotators
|
strict: false
|
marks:
|
- usefixtures:
|
- django_live_url
|
stages:
|
|
- id: signup
|
type: ref
|
|
- id: create_project
|
name: create_project
|
request:
|
data:
|
title: Test Draft 1
|
show_collab_predictions: true
|
label_config: '<View><Text name="text" value="$text"/><Choices name="text_class" toName="text"><Choice value="class_A"/><Choice value="class_B"/><Choice value="class_TESTING"/><Choice value="class_PREDICTIONS_TESTING"/></Choices></View>'
|
method: POST
|
url: '{django_live_url}/api/projects'
|
response:
|
save:
|
json:
|
project_pk: id
|
created_by: created_by.id
|
status_code: 201
|
|
- name: create_filter_tasks
|
request:
|
files:
|
json_file: tests/data_manager/tasks_annotations_predictions.json
|
headers:
|
content-type: multipart/form-data
|
method: POST
|
url: '{django_live_url}/api/projects/{project_pk}/import'
|
response:
|
json:
|
annotation_count: 4
|
prediction_count: 3
|
task_count: 6
|
status_code: 201
|
|
- name: setup_views
|
request:
|
method: POST
|
url: '{django_live_url}/api/dm/views/?project={project_pk}'
|
json: {
|
"data": {
|
"filters": {
|
"conjunction": "or",
|
"items": [
|
{
|
"filter": "filter:tasks:annotators",
|
"operator": "not_contains",
|
"type": "List",
|
"value": "{created_by}"
|
}
|
]
|
},
|
"ordering": [
|
"tasks:data.text"
|
]
|
},
|
"project": !int "{project_pk}"
|
}
|
response:
|
save:
|
json:
|
view: "@"
|
status_code: 201
|
|
- name: get_tasks
|
request:
|
method: GET
|
url: '{django_live_url}/api/tasks?fields=all&view={view.id}'
|
response:
|
status_code: 200
|
verify_response_with:
|
function: label_studio.tests.utils:save_response
|
json: {
|
"total_annotations": 0,
|
"total_predictions": 1,
|
"total": 3,
|
"tasks": [
|
{
|
"cancelled_annotations": 0,
|
"storage_filename": null,
|
|
"annotations_results": "",
|
"data": {
|
"int_field": 42,
|
"text": "opop"
|
},
|
|
|
"predictions_score": null,
|
"total_annotations": 0,
|
"total_predictions": 1,
|
"annotations_ids": !anystr "",
|
"annotations": [],
|
"predictions": [
|
{
|
|
"model_version": "undefined",
|
"created_ago": "0\u00a0minutes",
|
"result": [
|
{
|
"from_name": "text_class",
|
"to_name": "text",
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_PREDICTIONS_TESTING"
|
]
|
}
|
}
|
],
|
"score": null,
|
"cluster": null,
|
"neighbors": null,
|
"mislabeling": 0.0,
|
|
|
}
|
],
|
"drafts": [],
|
|
"annotators": [],
|
"project": !int "{project_pk}"
|
},
|
{
|
"cancelled_annotations": 0,
|
"storage_filename": null,
|
|
"annotations_results": "",
|
"data": {
|
"text": "vivi"
|
},
|
|
"predictions_results": "",
|
"predictions_score": null,
|
"total_annotations": 0,
|
"total_predictions": 0,
|
"annotations_ids": !anystr "",
|
"annotations": [],
|
"predictions": [],
|
"drafts": [],
|
|
"annotators": [],
|
"project": !int "{project_pk}"
|
},
|
{
|
"cancelled_annotations": 0,
|
"storage_filename": null,
|
|
"annotations_results": "",
|
"data": {
|
"text": "zzz"
|
},
|
|
"predictions_results": "",
|
"predictions_score": null,
|
"total_annotations": 0,
|
"total_predictions": 0,
|
"annotations_ids": !anystr "",
|
"annotations": [],
|
"predictions": [],
|
"drafts": [],
|
|
"annotators": [],
|
"project": !int "{project_pk}"
|
}
|
]
|
}
|
|
- name: setup_views
|
request:
|
method: POST
|
url: '{django_live_url}/api/dm/views/?project={project_pk}'
|
json: {
|
"data": {
|
"filters": {
|
"conjunction": "or",
|
"items": [
|
{
|
"filter": "filter:tasks:annotators",
|
"operator": "contains",
|
"type": "List",
|
"value": "{created_by}"
|
}
|
]
|
},
|
"ordering": [
|
"tasks:data.text"
|
]
|
},
|
"project": !int "{project_pk}"
|
}
|
response:
|
save:
|
json:
|
view: "@"
|
status_code: 201
|
|
- name: get_tasks_after_setup
|
request:
|
method: GET
|
url: '{django_live_url}/api/tasks?fields=all&view={view.id}'
|
response:
|
status_code: 200
|
verify_response_with:
|
function: label_studio.tests.utils:save_response
|
json: {
|
"total_annotations": 4,
|
"total_predictions": 2,
|
"total": 3,
|
"tasks": [
|
{
|
"cancelled_annotations": 0,
|
"storage_filename": null,
|
|
"data": {
|
"text": "Test example phrase 1",
|
"int_field": 1
|
},
|
|
|
"predictions_score": null,
|
"total_annotations": 1,
|
"total_predictions": 1,
|
"annotations_ids": !anystr "",
|
"annotations": [
|
{
|
|
"created_username": " test_suites_user@heartex.com, {created_by}",
|
"created_ago": "0\u00a0minutes",
|
"completed_by": !int "{created_by}",
|
"result": [
|
{
|
"from_name": "text_class",
|
"to_name": "text",
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_A"
|
]
|
}
|
}
|
],
|
"was_cancelled": false,
|
"ground_truth": true,
|
|
"lead_time": 3.0
|
|
}
|
],
|
"predictions": [
|
{
|
|
"model_version": "model_version_A",
|
"created_ago": "0\u00a0minutes",
|
"result": [
|
{
|
"from_name": "text_class",
|
"to_name": "text",
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_A"
|
]
|
}
|
}
|
],
|
"score": null,
|
"cluster": null,
|
"neighbors": null,
|
"mislabeling": 0.0,
|
|
|
}
|
],
|
"drafts": [],
|
|
"annotators": [
|
!int "{created_by}"
|
],
|
"project": !int "{project_pk}"
|
},
|
{
|
"cancelled_annotations": 0,
|
"storage_filename": null,
|
|
"data": {
|
"int_field": 20,
|
"text": "x2"
|
},
|
|
|
"predictions_score": null,
|
"total_annotations": 2,
|
"total_predictions": 1,
|
"annotations_ids": !anystr "",
|
"annotations": [
|
{
|
|
"created_username": " test_suites_user@heartex.com, {created_by}",
|
"created_ago": "0\u00a0minutes",
|
"completed_by": !int "{created_by}",
|
"result": [
|
{
|
"from_name": "text_class",
|
"to_name": "text",
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_A"
|
]
|
}
|
}
|
],
|
"was_cancelled": false,
|
"ground_truth": true,
|
|
"lead_time": 3.0
|
|
},
|
{
|
|
"created_username": " test_suites_user@heartex.com, {created_by}",
|
"created_ago": "0\u00a0minutes",
|
"completed_by": !int "{created_by}",
|
"result": [
|
{
|
"from_name": "text_class",
|
"to_name": "text",
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_A"
|
]
|
}
|
}
|
],
|
"was_cancelled": false,
|
"ground_truth": true,
|
|
"lead_time": 5.0
|
|
}
|
],
|
"predictions": [
|
{
|
|
"model_version": "model_version_A",
|
"created_ago": "0\u00a0minutes",
|
"result": [
|
{
|
"from_name": "text_class",
|
"to_name": "text",
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_A"
|
]
|
}
|
}
|
],
|
"score": null,
|
"cluster": null,
|
"neighbors": null,
|
"mislabeling": 0.0,
|
|
}
|
],
|
"drafts": [],
|
"annotators": [
|
!int "{created_by}"
|
],
|
"project": !int "{project_pk}"
|
},
|
{
|
"cancelled_annotations": 0,
|
"storage_filename": null,
|
|
"data": {
|
"int_field": "99",
|
"text": "yoyo"
|
},
|
|
"predictions_results": "",
|
"predictions_score": null,
|
"total_annotations": 1,
|
"total_predictions": 0,
|
"annotations_ids": !anystr "",
|
"annotations": [
|
{
|
|
"created_username": " test_suites_user@heartex.com, {created_by}",
|
"created_ago": "0\u00a0minutes",
|
"completed_by": !int "{created_by}",
|
"result": [
|
{
|
"from_name": "text_class",
|
"to_name": "text",
|
"type": "choices",
|
"value": {
|
"choices": [
|
"class_TESTING"
|
]
|
}
|
}
|
],
|
"was_cancelled": false,
|
"ground_truth": true,
|
|
"lead_time": null
|
|
}
|
],
|
"predictions": [],
|
"drafts": [],
|
|
"annotators": [
|
!int "{created_by}"
|
],
|
"project": !int "{project_pk}"
|
}
|
]
|
}
|
|
---
|
test_name: tasks-minimal-fields
|
strict: false
|
marks:
|
- usefixtures:
|
- django_live_url
|
stages:
|
|
- id: signup
|
type: ref
|
|
- id: create_project
|
name: create_project
|
request:
|
data:
|
title: Test Draft 1
|
show_collab_predictions: true
|
label_config: '<View><Text name="text" value="$text"/><Choices name="text_class" toName="text"><Choice value="class_A"/><Choice value="class_B"/><Choice value="class_TESTING"/><Choice value="class_PREDICTIONS_TESTING"/></Choices></View>'
|
method: POST
|
url: '{django_live_url}/api/projects'
|
response:
|
save:
|
json:
|
project_pk: id
|
created_by: created_by.id
|
status_code: 201
|
|
- name: create_filter_tasks
|
request:
|
files:
|
json_file: tests/data_manager/tasks_annotations_predictions.json
|
headers:
|
content-type: multipart/form-data
|
method: POST
|
url: '{django_live_url}/api/projects/{project_pk}/import'
|
response:
|
json:
|
annotation_count: 4
|
prediction_count: 3
|
task_count: 6
|
status_code: 201
|
|
- name: setup_views
|
request:
|
method: POST
|
url: '{django_live_url}/api/dm/views/?project={project_pk}'
|
json: {
|
"data": {
|
"filters": {
|
"conjunction": "or",
|
"items": [
|
{
|
"filter": "filter:tasks:annotators",
|
"operator": "empty",
|
"type": "List",
|
"value": false
|
}
|
]
|
},
|
"ordering": [
|
"tasks:data.text"
|
]
|
},
|
"project": !int "{project_pk}"
|
}
|
response:
|
save:
|
json:
|
view: "@"
|
status_code: 201
|
|
- name: get_tasks
|
request:
|
method: GET
|
url: '{django_live_url}/api/tasks?view={view.id}'
|
response:
|
status_code: 200
|
verify_response_with:
|
function: label_studio.tests.utils:save_response
|
json: {
|
"total_annotations": 4,
|
"total_predictions": 2,
|
"total": 3,
|
"tasks": [
|
{
|
|
"annotations_results": "",
|
"data": {
|
"text": "Test example phrase 1",
|
"int_field": 1
|
},
|
|
"predictions_results": "",
|
"total_annotations": 1,
|
"annotations_ids": !anystr "",
|
"drafts": [],
|
"file_upload": null,
|
"annotators": [
|
!int "{created_by}"
|
],
|
"project": !int "{project_pk}"
|
},
|
{
|
|
"annotations_results": "",
|
"data": {
|
"int_field": 20,
|
"text": "x2"
|
},
|
|
"predictions_results": "",
|
"total_annotations": 2,
|
"annotations_ids": !anystr "",
|
"drafts": [],
|
"file_upload": null,
|
"annotators": [
|
!int "{created_by}"
|
],
|
"project": !int "{project_pk}"
|
},
|
{
|
|
"annotations_results": "",
|
"data": {
|
"int_field": "99",
|
"text": "yoyo"
|
},
|
|
"predictions_results": "",
|
"total_annotations": 1,
|
"annotations_ids": !anystr "",
|
"drafts": [],
|
"file_upload": null,
|
"annotators": [
|
!int "{created_by}"
|
],
|
"project": !int "{project_pk}"
|
}
|
]
|
}
|
|
- name: setup_views
|
request:
|
method: POST
|
url: '{django_live_url}/api/dm/views/?project={project_pk}'
|
json: {
|
"data": {
|
"filters": {
|
"conjunction": "and",
|
"items": [
|
{
|
"filter": "filter:tasks:predictions_results",
|
"operator": "equal",
|
"type": "String",
|
"value": "PREDICTIONS"
|
},
|
{
|
"filter": "filter:tasks:annotations_results",
|
"operator": "not_equal",
|
"type": "String",
|
"value": "PREDICTIONS"
|
}
|
]
|
},
|
"ordering": [
|
"tasks:data.text"
|
]
|
},
|
"project": !int "{project_pk}"
|
}
|
response:
|
save:
|
json:
|
view: "@"
|
status_code: 201
|
|
- name: get_tasks
|
request:
|
method: GET
|
url: '{django_live_url}/api/tasks?view={view.id}'
|
response:
|
status_code: 200
|
verify_response_with:
|
function: label_studio.tests.utils:save_response
|
json: {
|
"total_annotations": 0,
|
"total_predictions": 0,
|
"total": 0,
|
}
|
|
- name: get_tasks_with_page
|
request:
|
method: GET
|
url: '{django_live_url}/api/tasks?view={view.id}&page=1'
|
response:
|
status_code: 200
|
verify_response_with:
|
function: label_studio.tests.utils:save_response
|
json: {
|
"total_annotations": 0,
|
"total_predictions": 0,
|
"total": 0,
|
}
|
|
- name: get_first_task
|
request:
|
method: GET
|
url: '{django_live_url}/api/projects/{project_pk}/next'
|
response:
|
save:
|
json:
|
first_task_id: id
|
status_code: 200
|
|
- name: create_annotation_first_task
|
request:
|
headers:
|
content-type: application/json
|
json:
|
lead_time: 5
|
result:
|
- from_name: label
|
to_name: text
|
type: choices
|
value:
|
choices:
|
- class_B
|
method: POST
|
url: '{django_live_url}/api/tasks/{first_task_id}/annotations'
|
response:
|
save:
|
json:
|
annotation_pk: id
|
status_code: 201
|
|
- name: setup_views
|
request:
|
method: POST
|
url: '{django_live_url}/api/dm/views/?project={project_pk}'
|
json: {
|
"data": {
|
"filters": {
|
"conjunction": "or",
|
"items": [
|
{
|
"filter": "filter:tasks:updated_by",
|
"operator": "contains",
|
"type": "List",
|
"value": !int "{created_by}",
|
}
|
]
|
},
|
"ordering": [
|
"tasks:data.text"
|
]
|
},
|
"project": !int "{project_pk}"
|
}
|
response:
|
save:
|
json:
|
view: "@"
|
status_code: 201
|
|
- name: get_tasks
|
request:
|
method: GET
|
url: '{django_live_url}/api/tasks?view={view.id}'
|
response:
|
status_code: 200
|
verify_response_with:
|
function: label_studio.tests.utils:save_response
|
json: {
|
"total_annotations": 1,
|
"total_predictions": 0,
|
"total": 1,
|
}
|
|
- name: setup_views
|
request:
|
method: POST
|
url: '{django_live_url}/api/dm/views/?project={project_pk}'
|
json: {
|
"data": {
|
"filters": {
|
"conjunction": "or",
|
"items": [
|
{
|
"filter": "filter:tasks:updated_by",
|
"operator": "not_contains",
|
"type": "List",
|
"value": !int "{created_by}",
|
}
|
]
|
},
|
"ordering": [
|
"tasks:data.text"
|
]
|
},
|
"project": !int "{project_pk}"
|
}
|
response:
|
save:
|
json:
|
view: "@"
|
status_code: 201
|
|
- name: get_tasks
|
request:
|
method: GET
|
url: '{django_live_url}/api/tasks?view={view.id}'
|
response:
|
status_code: 200
|
verify_response_with:
|
function: label_studio.tests.utils:save_response
|
json: {
|
"total_annotations": 4,
|
"total_predictions": 3,
|
"total": 5,
|
}
|
|
- name: setup_views
|
request:
|
method: POST
|
url: '{django_live_url}/api/dm/views/?project={project_pk}'
|
json: {
|
"data": {
|
"filters": {
|
"conjunction": "or",
|
"items": [
|
{
|
"filter": "filter:tasks:updated_by",
|
"operator": "equal",
|
"type": "List",
|
"value": !int "{created_by}",
|
}
|
]
|
},
|
"ordering": [
|
"tasks:data.text"
|
]
|
},
|
"project": !int "{project_pk}"
|
}
|
response:
|
save:
|
json:
|
view: "@"
|
status_code: 201
|
|
- name: get_tasks
|
request:
|
method: GET
|
url: '{django_live_url}/api/tasks?view={view.id}'
|
response:
|
status_code: 200
|
verify_response_with:
|
function: label_studio.tests.utils:save_response
|
json: {
|
"total_annotations": 1,
|
"total_predictions": 0,
|
"total": 1,
|
}
|
|
---
|
|
test_name: tasks-storage-filename-s3
|
strict: false
|
marks:
|
- usefixtures:
|
- django_live_url
|
stages:
|
|
- id: signup
|
type: ref
|
|
- id: create_project
|
name: create_project
|
request:
|
data:
|
title: Test Draft 1
|
show_collab_predictions: true
|
label_config: '<View><Text name="text" value="$text"/><Choices name="text_class" toName="text"><Choice value="class_A"/><Choice value="class_B"/><Choice value="class_TESTING"/><Choice value="class_PREDICTIONS_TESTING"/></Choices></View>'
|
method: POST
|
url: '{django_live_url}/api/projects'
|
response:
|
save:
|
json:
|
project_pk: id
|
created_by: created_by.id
|
status_code: 201
|
|
- name: Add S3 storage
|
request:
|
data:
|
bucket: pytest-s3-images
|
project: '{project_pk}'
|
title: Testing S3 storage (bucket from conftest.py)
|
use_blob_urls: true
|
presign_ttl: 3600
|
recursive_scan: true
|
method: POST
|
url: '{django_live_url}/api/storages/s3'
|
response:
|
save:
|
json:
|
storage_pk: id
|
status_code: 201
|
|
- name: Sync S3 storage
|
request:
|
method: POST
|
url: '{django_live_url}/api/storages/s3/{storage_pk}/sync'
|
response:
|
json:
|
last_sync_count: 4
|
status_code: 200
|
|
- name: setup_views
|
request:
|
method: POST
|
url: '{django_live_url}/api/dm/views/?project={project_pk}'
|
json: {
|
"data": {
|
"filters": {},
|
"ordering": [
|
"tasks:storage_filename"
|
]
|
},
|
"project": !int "{project_pk}"
|
}
|
response:
|
save:
|
json:
|
view: "@"
|
status_code: 201
|
|
- name: get_tasks
|
request:
|
method: GET
|
url: '{django_live_url}/api/tasks?fields=all&view={view.id}'
|
response:
|
status_code: 200
|
json: {
|
"total_annotations": 0,
|
"total_predictions": 0,
|
"total": 4,
|
"tasks": [
|
{
|
"storage_filename": "image1.jpg",
|
"project": !int "{project_pk}"
|
},
|
{
|
"storage_filename": "subdir/another/image2.jpg",
|
"project": !int "{project_pk}"
|
},
|
{
|
"storage_filename": "subdir/image1.jpg",
|
"project": !int "{project_pk}"
|
},
|
{
|
"storage_filename": "subdir/image2.jpg",
|
"project": !int "{project_pk}"
|
}
|
]
|
}
|
|
---
|
|
test_name: tasks-storage-filename-local
|
strict: false
|
marks:
|
- skipif: '"Windows" in platform.system()'
|
- usefixtures:
|
- django_live_url
|
- local_files_storage
|
- local_files_document_root_tempdir
|
stages:
|
|
- id: signup
|
name: Sign up
|
request:
|
url: "{django_live_url}/user/signup"
|
data:
|
email: test_suites_user@heartex.com
|
password: 12345678
|
method: POST
|
response:
|
status_code: 302
|
save:
|
$ext:
|
function: tests.utils:os_independent_path
|
extra_kwargs:
|
path: 'files/subdir/test_image2.png'
|
add_tempdir: true
|
|
- id: create_project
|
name: create_project
|
request:
|
data:
|
title: Test Draft 1
|
show_collab_predictions: true
|
label_config: '<View><Text name="text" value="$text"/><Choices name="text_class" toName="text"><Choice value="class_A"/><Choice value="class_B"/><Choice value="class_TESTING"/><Choice value="class_PREDICTIONS_TESTING"/></Choices></View>'
|
method: POST
|
url: '{django_live_url}/api/projects'
|
response:
|
save:
|
json:
|
project_pk: id
|
created_by: created_by.id
|
status_code: 201
|
|
- name: create local storage
|
request:
|
json:
|
project: "{project_pk}"
|
title: test
|
regex_filter: ".*png"
|
path: "{os_independent_path_parent}"
|
use_blob_urls: true
|
method: POST
|
url: '{django_live_url}/api/storages/localfiles'
|
response:
|
save:
|
json:
|
storage_pk: id
|
status_code: 201
|
|
- name: sync local storage
|
request:
|
method: POST
|
url: '{django_live_url}/api/storages/localfiles/{storage_pk}/sync'
|
response:
|
json:
|
last_sync_count: 1
|
status_code: 200
|
|
- name: setup_views
|
request:
|
method: POST
|
url: '{django_live_url}/api/dm/views/?project={project_pk}'
|
json: {
|
"data": {
|
"filters":{
|
"conjunction": "and",
|
"items": [{
|
"filter": "filter:tasks:storage_filename",
|
"operator": "contains",
|
"type": "String",
|
"value": ".png"
|
}]
|
},
|
"ordering": []
|
},
|
"project": !int "{project_pk}"
|
}
|
response:
|
save:
|
json:
|
view: "@"
|
status_code: 201
|
|
- name: get_tasks
|
request:
|
method: GET
|
url: '{django_live_url}/api/tasks?fields=all&view={view.id}'
|
response:
|
status_code: 200
|
json: {
|
"total_annotations": 0,
|
"total_predictions": 0,
|
"total": 1,
|
"tasks": [
|
{
|
"storage_filename": "/tmp/files/subdir/test_image2.png",
|
"project": !int "{project_pk}"
|
}
|
]
|
}
|
|
---
|
|
# See FilterSerializer; test logic preventing exploit that traverses the ORM
|
# to leak sensitive data character-by-character
|
test_name: tasks_api_filter_security_restrictions
|
strict: false
|
marks:
|
- usefixtures:
|
- django_live_url
|
stages:
|
|
- id: signup
|
type: ref
|
|
- id: create_project
|
name: create_project
|
request:
|
data:
|
title: Test Draft 1
|
show_collab_predictions: true
|
label_config: '<View><Text name="text" value="$text"/><Choices name="text_class" toName="text"><Choice value="class_A"/><Choice value="class_B"/><Choice value="class_TESTING"/><Choice value="class_PREDICTIONS_TESTING"/></Choices></View>'
|
method: POST
|
url: '{django_live_url}/api/projects'
|
response:
|
save:
|
json:
|
project_pk: id
|
created_by: created_by.id
|
status_code: 201
|
|
- name: create view with exploit filter should fail
|
request:
|
method: POST
|
url: '{django_live_url}/api/dm/views'
|
json:
|
project: '{project_pk}'
|
data:
|
filters:
|
conjunction: and
|
items:
|
- filter: "filter:tasks:updated_by__sensitive_field"
|
operator: empty
|
value: "true"
|
type: String
|
response:
|
status_code: 400
|
json:
|
validation_errors:
|
filter_group:
|
filters:
|
- column:
|
- '"__" is not generally allowed in filters. Consider asking your administrator to add "updated_by__sensitive_field" to DATA_MANAGER_FILTER_ALLOWLIST, but note that some filter expressions may pose a security risk'
|
|
- name: create view with allowlisted filter should succeed
|
request:
|
method: POST
|
url: '{django_live_url}/api/dm/views'
|
json:
|
project: '{project_pk}'
|
data:
|
filters:
|
conjunction: and
|
items:
|
- filter: "filter:tasks:updated_by__active_organization"
|
operator: empty
|
value: "true"
|
type: String
|
response:
|
status_code: 201
|
save:
|
json:
|
view_pk: id
|
|
- name: change filter to typical column succeeds
|
request:
|
method: PUT
|
url: '{django_live_url}/api/dm/views/{view_pk}?interaction=filter&project={project_pk}'
|
json:
|
project: '{project_pk}'
|
data:
|
filters:
|
conjunction: and
|
items:
|
- filter: "filter:tasks:updated_by"
|
operator: empty
|
value: "true"
|
type: String
|
response:
|
status_code: 200
|
|
- name: change filter to unexpected prefix fails
|
request:
|
method: PUT
|
url: '{django_live_url}/api/dm/views/{view_pk}?interaction=filter&project={project_pk}'
|
json:
|
project: '{project_pk}'
|
data:
|
filters:
|
conjunction: and
|
items:
|
- filter: "tasks:filter:annotations_results"
|
operator: empty
|
value: "true"
|
type: String
|
response:
|
status_code: 400
|
json:
|
validation_errors:
|
filter_group:
|
filters:
|
- column:
|
- 'Filter "tasks:filter:annotations_results" should start with "filter:tasks:"'
|
|
- name: change filter to include direction marker succeeds
|
request:
|
method: PUT
|
url: '{django_live_url}/api/dm/views/{view_pk}?interaction=filter&project={project_pk}'
|
json:
|
project: '{project_pk}'
|
data:
|
filters:
|
conjunction: and
|
items:
|
- filter: "filter:tasks:-annotations_results"
|
operator: empty
|
value: "true"
|
type: String
|
response:
|
status_code: 200
|
|
- name: invalid filter with direction marker fails, but suggests correct allowlist
|
request:
|
method: PUT
|
url: '{django_live_url}/api/dm/views/{view_pk}?interaction=filter&project={project_pk}'
|
json:
|
project: '{project_pk}'
|
data:
|
filters:
|
conjunction: and
|
items:
|
- filter: "filter:tasks:-updated_by__sensitive_field"
|
operator: empty
|
value: "true"
|
type: String
|
response:
|
status_code: 400
|
json:
|
validation_errors:
|
filter_group:
|
filters:
|
- column:
|
- '"__" is not generally allowed in filters. Consider asking your administrator to add "updated_by__sensitive_field" to DATA_MANAGER_FILTER_ALLOWLIST, but note that some filter expressions may pose a security risk'
|
|
- name: change filter to dotted data field with underscores succeeds
|
request:
|
method: PUT
|
url: '{django_live_url}/api/dm/views/{view_pk}?interaction=filter&project={project_pk}'
|
json:
|
project: '{project_pk}'
|
data:
|
filters:
|
conjunction: and
|
items:
|
- filter: "filter:tasks:data.images__0"
|
operator: empty
|
value: "true"
|
type: String
|
response:
|
status_code: 200
|
|
- name: change filter to valid column without prefix fails
|
request:
|
method: PUT
|
url: '{django_live_url}/api/dm/views/{view_pk}?interaction=filter&project={project_pk}'
|
json:
|
project: '{project_pk}'
|
data:
|
filters:
|
conjunction: and
|
items:
|
- filter: "annotations_results"
|
operator: empty
|
value: "true"
|
type: String
|
response:
|
status_code: 400
|
json:
|
validation_errors:
|
filter_group:
|
filters:
|
- column:
|
- 'Filter "annotations_results" should start with "filter:tasks:"'
|
|
---
|
|
# Regression test preventing replace-all of prefixes from being used to construct exploit filters
|
test_name: tasks_api_prevent_prefix_regression
|
strict: false
|
marks:
|
- usefixtures:
|
- django_live_url
|
stages:
|
|
- id: signup
|
type: ref
|
|
- id: create_project
|
name: create_project
|
request:
|
data:
|
title: Test Draft 1
|
show_collab_predictions: true
|
label_config: '<View><Text name="text" value="$text"/><Choices name="text_class" toName="text"><Choice value="class_A"/><Choice value="class_B"/><Choice value="class_TESTING"/><Choice value="class_PREDICTIONS_TESTING"/></Choices></View>'
|
method: POST
|
url: '{django_live_url}/api/projects'
|
response:
|
save:
|
json:
|
project_pk: id
|
created_by: created_by.id
|
status_code: 201
|
|
# Perhaps our validation should be further improved, but for now, it's
|
# garbage in, garbage out.
|
- name: create view with prefix in middle of filter should succeed
|
request:
|
method: POST
|
url: '{django_live_url}/api/dm/views'
|
json:
|
project: '{project_pk}'
|
data:
|
filters:
|
conjunction: and
|
items:
|
# in an earlier version, preprocess_field_name would have
|
# transformed this to updated_by__sensitive_field
|
- filter: "filter:tasks:updated_by_tasks:_sensitive_field"
|
operator: empty
|
value: "true"
|
type: String
|
response:
|
status_code: 201
|
save:
|
json:
|
view_pk: id
|
|
- name: get_tasks fails because "updated_by_tasks:_sensitive_field" doesn't resolve to a field
|
request:
|
method: GET
|
url: '{django_live_url}/api/tasks?view={view_pk}'
|
response:
|
# 500 actually expected here
|
status_code: 500
|
