cayu.evals.execution

Public Cayu API. Definitions are grouped by their public source module, and definitions from private implementation modules appear on their package’s page; adding a symbol does not move existing links. The import shown at each entry is a supported public namespace. Read the Python API overview and install the required extra.

compile_corpus_suite

from cayu.evals import compile_corpus_suite

Compile authority-free corpus data against one trusted local target.

compile_corpus_suite(corpus: EvalCorpusDocument, target: CorpusTarget, suite_id: str) -> CompiledCorpusSuite

Release source · Runtime guarantees · Capability guides

CompiledCorpusSuite

from cayu.evals import CompiledCorpusSuite

One validated corpus suite ready for the existing evaluator.

Decorators: @dataclass(frozen=True).

Fields and attributes

Defaults below are source expressions; Field(...) includes schema/validation settings and default_factory creates a fresh value. Inherited declarations are included. Constructor behavior follows the model or dataclass shown in the bases/decorators.

Name	Type	Default
corpus	EvalCorpusDocument	required
suite	EvalSuite	required
run_contract	EvalRunContractV2	required
trials	int	required
timeout_seconds	int	required

Release source · Runtime guarantees · Capability guides

CORPUS_EXECUTION_DEFAULT_MAX_CONCURRENCY

from cayu.evals import CORPUS_EXECUTION_DEFAULT_MAX_CONCURRENCY

CORPUS_EXECUTION_DEFAULT_MAX_CONCURRENCY = DEFAULT_EVAL_MAX_ACTIVE_TRIALS

Release source · Runtime guarantees · Capability guides

CORPUS_EXECUTION_MAX_APP_MANIFEST_BYTES

from cayu.evals import CORPUS_EXECUTION_MAX_APP_MANIFEST_BYTES

CORPUS_EXECUTION_MAX_APP_MANIFEST_BYTES = 1 << 20

Release source · Runtime guarantees · Capability guides

CORPUS_EXECUTION_MAX_BOOTSTRAP_MESSAGES

from cayu.evals import CORPUS_EXECUTION_MAX_BOOTSTRAP_MESSAGES

CORPUS_EXECUTION_MAX_BOOTSTRAP_MESSAGES = 128

Release source · Runtime guarantees · Capability guides

CORPUS_EXECUTION_MAX_COMPILED_INPUT_CHARS

from cayu.evals import CORPUS_EXECUTION_MAX_COMPILED_INPUT_CHARS

CORPUS_EXECUTION_MAX_COMPILED_INPUT_CHARS = 8 << 20

Release source · Runtime guarantees · Capability guides

CORPUS_EXECUTION_MAX_CONCURRENCY

from cayu.evals import CORPUS_EXECUTION_MAX_CONCURRENCY

CORPUS_EXECUTION_MAX_CONCURRENCY = CORPUS_EXECUTION_DEFAULT_MAX_CONCURRENCY

Release source · Runtime guarantees · Capability guides

CORPUS_EXECUTION_MAX_MODEL_JUDGES

from cayu.evals import CORPUS_EXECUTION_MAX_MODEL_JUDGES

CORPUS_EXECUTION_MAX_MODEL_JUDGES = 32

Release source · Runtime guarantees · Capability guides

CORPUS_EXECUTION_MAX_REQUEST_BASE_BYTES

from cayu.evals import CORPUS_EXECUTION_MAX_REQUEST_BASE_BYTES

CORPUS_EXECUTION_MAX_REQUEST_BASE_BYTES = 64 << 10

Release source · Runtime guarantees · Capability guides

CORPUS_EXECUTION_MAX_TOTAL_INPUT_CHARS

from cayu.evals import CORPUS_EXECUTION_MAX_TOTAL_INPUT_CHARS

CORPUS_EXECUTION_MAX_TOTAL_INPUT_CHARS = EVAL_CORPUS_MAX_TOTAL_MESSAGE_CHARS * 2

Release source · Runtime guarantees · Capability guides

CORPUS_EXECUTION_RESULT_MAX_BYTES

from cayu.evals import CORPUS_EXECUTION_RESULT_MAX_BYTES

CORPUS_EXECUTION_RESULT_MAX_BYTES = 40 << 20

Release source · Runtime guarantees · Capability guides

CORPUS_EXECUTION_RESULT_SCHEMA_VERSION

from cayu.evals import CORPUS_EXECUTION_RESULT_SCHEMA_VERSION

CORPUS_EXECUTION_RESULT_SCHEMA_VERSION = 3

Release source · Runtime guarantees · Capability guides

CorpusExecutionLimits

from cayu.evals import CorpusExecutionLimits

Trusted ceilings that may narrow, but never expand, portable corpus limits.

Bases: BaseModel.

Fields and attributes

Defaults below are source expressions; Field(...) includes schema/validation settings and default_factory creates a fresh value. Inherited declarations are included. Constructor behavior follows the model or dataclass shown in the bases/decorators.

Name	Type	Default
max_cases	StrictInt	Field(default=EVAL_CORPUS_MAX_CASES, ge=1, le=EVAL_CORPUS_MAX_CASES)
max_trials	StrictInt	Field(default=EVAL_CORPUS_MAX_TRIALS, ge=1, le=EVAL_CORPUS_MAX_TRIALS)
max_timeout_seconds	StrictInt	Field(default=EVAL_CORPUS_MAX_TIMEOUT_SECONDS, ge=1, le=EVAL_CORPUS_MAX_TIMEOUT_SECONDS)
max_concurrency	StrictInt	Field(default=CORPUS_EXECUTION_DEFAULT_MAX_CONCURRENCY, ge=1, le=EVAL_MAX_CONCURRENCY)
max_bootstrap_messages	StrictInt	Field(default=EVAL_CORPUS_MAX_MESSAGES_PER_CASE, ge=0, le=CORPUS_EXECUTION_MAX_BOOTSTRAP_MESSAGES)
max_total_input_chars	StrictInt	Field(default=CORPUS_EXECUTION_MAX_TOTAL_INPUT_CHARS, ge=1, le=CORPUS_EXECUTION_MAX_TOTAL_INPUT_CHARS)
max_compiled_input_chars	StrictInt	Field(default=CORPUS_EXECUTION_MAX_COMPILED_INPUT_CHARS, ge=1, le=CORPUS_EXECUTION_MAX_COMPILED_INPUT_CHARS)

Release source · Runtime guarantees · Capability guides

CorpusExecutionResult

from cayu.evals import CorpusExecutionResult

Safe published run plus the fresh target identity used to produce it.

Bases: BaseModel.

Fields and attributes

Defaults below are source expressions; Field(...) includes schema/validation settings and default_factory creates a fresh value. Inherited declarations are included. Constructor behavior follows the model or dataclass shown in the bases/decorators.

Name	Type	Default
launch_scheduling	LaunchScheduling | None	Field(default=None, exclude_if=lambda value: value is None)
schema_version	Literal[3]	CORPUS_EXECUTION_RESULT_SCHEMA_VERSION
revision	StrictStr	required
target	EvaluationTargetIdentity	required
run	PublishedEvalRun	required
external_trials	tuple[ExternalTrialIdentityV1, ...]	Field(default=(), max_length=EVAL_CORPUS_MAX_CASES * EVAL_CORPUS_MAX_TRIALS, exclude_if=lambda value: not value)

validate_schema_version_type (method)

CorpusExecutionResult.validate_schema_version_type(value: object) -> object

validate_revision_shape (method)

CorpusExecutionResult.validate_revision_shape(value: str, info) -> str

copy_target (method)

CorpusExecutionResult.copy_target(value: object) -> object

copy_run (method)

CorpusExecutionResult.copy_run(value: object) -> object

copy_external_trials (method)

CorpusExecutionResult.copy_external_trials(value: object) -> object

validate_contract (method)

CorpusExecutionResult.validate_contract() -> CorpusExecutionResult

create (method)

CorpusExecutionResult.create(*, target: EvaluationTargetIdentity, run: PublishedEvalRun, external_trials: tuple[ExternalTrialIdentityV1, ...]=(), launch_scheduling: LaunchScheduling | None=None) -> CorpusExecutionResult

Release source · Runtime guarantees · Capability guides

CorpusTarget

from cayu.evals import CorpusTarget

Trusted local execution authority selected by an eval project target.

Bases: BaseModel.

Fields and attributes

Defaults below are source expressions; Field(...) includes schema/validation settings and default_factory creates a fresh value. Inherited declarations are included. Constructor behavior follows the model or dataclass shown in the bases/decorators.

Name	Type	Default
key	StrictStr	required
app	CayuApp	required
request_base	RunRequest	required
bootstrap_messages	tuple[Message, ...]	()
application_release_id	StrictStr	required
evidence_policy	EvaluationEvidencePolicySpec	Field(default_factory=EvaluationEvidencePolicySpec.standard)
price_book	PriceBook | None	None
model_judges	tuple[ModelJudgeTarget, ...]	Field(default=(), max_length=CORPUS_EXECUTION_MAX_MODEL_JUDGES)
limits	CorpusExecutionLimits	Field(default_factory=CorpusExecutionLimits)
external_process	ExternalProcessTargetIdentityV1 | None	Field(default=None, exclude=True)

validate_key (method)

CorpusTarget.validate_key(value: str) -> str

validate_app (method)

CorpusTarget.validate_app(value: CayuApp) -> CayuApp

validate_request_base (method)

CorpusTarget.validate_request_base(value: object) -> RunRequest

validate_bootstrap_messages (method)

CorpusTarget.validate_bootstrap_messages(value: object) -> tuple[Message, ...]

validate_application_release_id (method)

CorpusTarget.validate_application_release_id(value: str, info) -> str

copy_evidence_policy (method)

CorpusTarget.copy_evidence_policy(value: object) -> EvaluationEvidencePolicySpec

copy_trusted_price_book (method)

CorpusTarget.copy_trusted_price_book(value: object) -> PriceBook | None

copy_model_judges (method)

CorpusTarget.copy_model_judges(value: object) -> tuple[ModelJudgeTarget, ...]

copy_limits (method)

CorpusTarget.copy_limits(value: object) -> CorpusExecutionLimits

copy_external_process (method)

CorpusTarget.copy_external_process(value: object) -> ExternalProcessTargetIdentityV1 | None

validate_authority_boundary (method)

CorpusTarget.validate_authority_boundary() -> CorpusTarget

Release source · Runtime guarantees · Capability guides

evaluation_target_identity

from cayu.evals import evaluation_target_identity

Describe a validated target without invoking application dependencies.

evaluation_target_identity(target: CorpusTarget, *, project_root: str | Path | None=None) -> EvaluationTargetIdentity

Release source · Runtime guarantees · Capability guides

EvaluationTargetIdentity

from cayu.evals import EvaluationTargetIdentity

Public diagnostic identity of the fresh application used for execution.

Bases: BaseModel.

Fields and attributes

Defaults below are source expressions; Field(...) includes schema/validation settings and default_factory creates a fresh value. Inherited declarations are included. Constructor behavior follows the model or dataclass shown in the bases/decorators.

Name	Type	Default
schema_version	Literal[1]	1
target_key	StrictStr	required
application_release_id	StrictStr	required
app_manifest	AppManifest	required
external_process	ExternalProcessTargetIdentityV1 | None	Field(default=None, exclude_if=lambda value: value is None)
workflow	WorkflowEvalTargetIdentityV1 | None	Field(default=None, exclude_if=lambda value: value is None)

validate_schema_version_type (method)

EvaluationTargetIdentity.validate_schema_version_type(value: object) -> object

validate_target_key (method)

EvaluationTargetIdentity.validate_target_key(value: str, info) -> str

validate_release_id (method)

EvaluationTargetIdentity.validate_release_id(value: str, info) -> str

copy_app_manifest (method)

EvaluationTargetIdentity.copy_app_manifest(value: object) -> object

copy_external_process (method)

EvaluationTargetIdentity.copy_external_process(value: object) -> object

copy_workflow (method)

EvaluationTargetIdentity.copy_workflow(value: object) -> object

validate_app_manifest (method)

EvaluationTargetIdentity.validate_app_manifest() -> EvaluationTargetIdentity

app_manifest_schema_version (property)

EvaluationTargetIdentity.app_manifest_schema_version() -> str

app_manifest_fingerprint (property)

EvaluationTargetIdentity.app_manifest_fingerprint() -> str

Release source · Runtime guarantees · Capability guides

model_judge_implementation_revision

from cayu.evals import model_judge_implementation_revision

Return the implementation identity a trusted target will publish.

model_judge_implementation_revision(target: ModelJudgeTarget) -> str

Release source · Runtime guarantees · Capability guides

model_judge_profile

from cayu.evals import model_judge_profile

Publish the bounded identity and ceilings of one explicit judge route.

model_judge_profile(target: ModelJudgeTarget) -> JudgeProfileIdentityV1

Release source · Runtime guarantees · Capability guides

ModelJudgeTarget

from cayu.evals import ModelJudgeTarget

Trusted local execution authority for one portable evaluator key.

Bases: BaseModel.

Fields and attributes

Defaults below are source expressions; Field(...) includes schema/validation settings and default_factory creates a fresh value. Inherited declarations are included. Constructor behavior follows the model or dataclass shown in the bases/decorators.

Name	Type	Default
key	StrictStr	required
app	CayuApp	required
agent_name	StrictStr	required
label	StrictStr | None	None
privacy_policy	JudgePrivacyPolicyV1	Field(default_factory=JudgePrivacyPolicyV1.public_only)
private_references	tuple[PrivateJudgeReferenceTarget, ...]	Field(default=(), max_length=MODEL_JUDGE_MAX_PRIVATE_REFERENCES)
timeout_seconds	StrictInt	Field(default=120, ge=1, le=EVAL_CORPUS_MAX_TIMEOUT_SECONDS)
max_input_tokens	StrictInt	Field(default=32768, ge=1, le=EVIDENCE_MAX_TOTAL_TOKENS)
max_output_tokens	StrictInt	Field(default=4096, ge=1, le=EVIDENCE_MAX_TOTAL_TOKENS)
max_total_tokens	StrictInt	Field(default=36864, ge=1, le=EVIDENCE_MAX_TOTAL_TOKENS)
max_estimated_cost	StrictStr | None	None
cost_currency	StrictStr	'USD'
price_book	PriceBook | None	None
allow_same_model	StrictBool	False

validate_key (method)

ModelJudgeTarget.validate_key(value: str) -> str

validate_app (method)

ModelJudgeTarget.validate_app(value: CayuApp) -> CayuApp

validate_agent_name (method)

ModelJudgeTarget.validate_agent_name(value: str, info) -> str

validate_label (method)

ModelJudgeTarget.validate_label(value: str | None, info) -> str | None

copy_privacy_policy (method)

ModelJudgeTarget.copy_privacy_policy(value: object) -> JudgePrivacyPolicyV1

copy_private_references (method)

ModelJudgeTarget.copy_private_references(value: object) -> tuple[PrivateJudgeReferenceTarget, ...]

validate_max_estimated_cost (method)

ModelJudgeTarget.validate_max_estimated_cost(value: str | None, info) -> str | None

validate_cost_currency (method)

ModelJudgeTarget.validate_cost_currency(value: str, info) -> str

copy_price_book (method)

ModelJudgeTarget.copy_price_book(value: object) -> PriceBook | None

validate_authority_boundary (method)

ModelJudgeTarget.validate_authority_boundary() -> ModelJudgeTarget

Release source · Runtime guarantees · Capability guides

PrivateJudgeReferenceTarget

from cayu.evals import PrivateJudgeReferenceTarget

Trusted evaluator-only reference content that never enters a corpus or result.

Bases: BaseModel.

Fields and attributes

Defaults below are source expressions; Field(...) includes schema/validation settings and default_factory creates a fresh value. Inherited declarations are included. Constructor behavior follows the model or dataclass shown in the bases/decorators.

Name	Type	Default
key	StrictStr	required
revision	StrictStr	required
content	StrictStr	Field(repr=False)
privacy_policy_key	StrictStr	required
privacy_policy_revision	StrictStr	required

validate_ids (method)

PrivateJudgeReferenceTarget.validate_ids(value: str, info) -> str

validate_revisions (method)

PrivateJudgeReferenceTarget.validate_revisions(value: str, info) -> str

validate_content (method)

PrivateJudgeReferenceTarget.validate_content(value: str, info) -> str

validate_content_revision (method)

PrivateJudgeReferenceTarget.validate_content_revision() -> PrivateJudgeReferenceTarget

create (method)

PrivateJudgeReferenceTarget.create(*, key: str, content: str, privacy_policy: JudgePrivacyPolicyV1) -> PrivateJudgeReferenceTarget

portable_identity (method)

PrivateJudgeReferenceTarget.portable_identity() -> PrivateJudgeReferenceV1

Release source · Runtime guarantees · Capability guides

run_corpus_suite

from cayu.evals import run_corpus_suite

Execute one corpus suite through Cayu’s existing runner and publish it safely.

async run_corpus_suite(target: CorpusTarget, corpus: EvalCorpusDocument, suite_id: str, *, max_concurrency: int | None=None, execution_capacity: EvalExecutionCapacity | None=None) -> CorpusExecutionResult

Release source · Runtime guarantees · Capability guides

WorkflowEvalTarget

from cayu.evals import WorkflowEvalTarget

Trusted application-owned workflow-root execution target.

request_base and bootstrap_messages remain the shared corpus input compiler contract. The request’s agent name is only a profile-probe identity; candidate execution is performed exclusively by workflow_factory.

Bases: CorpusTarget.

Fields and attributes

Defaults below are source expressions; Field(...) includes schema/validation settings and default_factory creates a fresh value. Inherited declarations are included. Constructor behavior follows the model or dataclass shown in the bases/decorators.

Name	Type	Default
key	StrictStr	required
app	CayuApp	required
request_base	RunRequest	required
bootstrap_messages	tuple[Message, ...]	()
application_release_id	StrictStr	required
evidence_policy	EvaluationEvidencePolicySpec	Field(default_factory=EvaluationEvidencePolicySpec.standard)
price_book	PriceBook | None	None
model_judges	tuple[ModelJudgeTarget, ...]	Field(default=(), max_length=CORPUS_EXECUTION_MAX_MODEL_JUDGES)
limits	CorpusExecutionLimits	Field(default_factory=CorpusExecutionLimits)
external_process	ExternalProcessTargetIdentityV1 | None	Field(default=None, exclude=True)
capture_bounds	SessionTrajectoryBounds	Field(default_factory=SessionTrajectoryBounds)
workflow_spec	WorkflowSpec	required
implementation_revision	StrictStr	required
result_projector_revision	StrictStr	required
execution_scope_revision	StrictStr	required
instance_scope	WorkflowEvalInstanceScope	WorkflowEvalInstanceScope.PER_TRIAL
workflow_factory	WorkflowEvalFactory	Field(exclude=True, repr=False)
result_projector	WorkflowEvalResultProjector	Field(exclude=True, repr=False)
application_context	dict[str, Any]	Field(default_factory=dict, exclude=True, repr=False)
close_timeout_seconds	float	WORKFLOW_EVAL_DEFAULT_CLOSE_TIMEOUT_SECONDS

validate_key (method)

WorkflowEvalTarget.validate_key(value: str) -> str

validate_app (method)

WorkflowEvalTarget.validate_app(value: CayuApp) -> CayuApp

validate_request_base (method)

WorkflowEvalTarget.validate_request_base(value: object) -> RunRequest

validate_bootstrap_messages (method)

WorkflowEvalTarget.validate_bootstrap_messages(value: object) -> tuple[Message, ...]

validate_application_release_id (method)

WorkflowEvalTarget.validate_application_release_id(value: str, info) -> str

copy_evidence_policy (method)

WorkflowEvalTarget.copy_evidence_policy(value: object) -> EvaluationEvidencePolicySpec

copy_trusted_price_book (method)

WorkflowEvalTarget.copy_trusted_price_book(value: object) -> PriceBook | None

copy_model_judges (method)

WorkflowEvalTarget.copy_model_judges(value: object) -> tuple[ModelJudgeTarget, ...]

copy_limits (method)

WorkflowEvalTarget.copy_limits(value: object) -> CorpusExecutionLimits

copy_external_process (method)

WorkflowEvalTarget.copy_external_process(value: object) -> ExternalProcessTargetIdentityV1 | None

validate_authority_boundary (method)

WorkflowEvalTarget.validate_authority_boundary() -> CorpusTarget

copy_workflow_spec (method)

WorkflowEvalTarget.copy_workflow_spec(value: object) -> WorkflowSpec

validate_workflow_revisions (method)

WorkflowEvalTarget.validate_workflow_revisions(value: str, info) -> str

validate_callbacks (method)

WorkflowEvalTarget.validate_callbacks(value: object, info) -> object

copy_application_context (method)

WorkflowEvalTarget.copy_application_context(value: object) -> dict[str, Any]

validate_close_timeout (method)

WorkflowEvalTarget.validate_close_timeout(value: object) -> float

validate_workflow_contract (method)

WorkflowEvalTarget.validate_workflow_contract() -> WorkflowEvalTarget

identity (method)

WorkflowEvalTarget.identity() -> WorkflowEvalTargetIdentityV1

Release source · Runtime guarantees · Capability guides