kiln_ai.adapters.prompt_builders

  1import json
  2from abc import ABCMeta, abstractmethod
  3from typing import Dict
  4
  5from kiln_ai.datamodel import Task, TaskRun
  6from kiln_ai.utils.formatting import snake_case
  7
  8
  9class BasePromptBuilder(metaclass=ABCMeta):
 10    """Base class for building prompts from tasks.
 11
 12    Provides the core interface and basic functionality for prompt builders.
 13    """
 14
 15    def __init__(self, task: Task):
 16        """Initialize the prompt builder with a task.
 17
 18        Args:
 19            task (Task): The task containing instructions and requirements.
 20        """
 21        self.task = task
 22
 23    def prompt_id(self) -> str | None:
 24        """Returns the ID of the prompt, scoped to this builder.
 25
 26        Returns:
 27            str | None: The ID of the prompt, or None if not set.
 28        """
 29        return None
 30
 31    def build_prompt(self, include_json_instructions) -> str:
 32        """Build and return the complete prompt string.
 33
 34        Returns:
 35            str: The constructed prompt.
 36        """
 37        prompt = self.build_base_prompt()
 38
 39        if include_json_instructions and self.task.output_schema():
 40            prompt = (
 41                prompt
 42                + f"\n\n# Format Instructions\n\nReturn a JSON object conforming to the following schema:\n```\n{self.task.output_schema()}\n```"
 43            )
 44
 45        return prompt
 46
 47    @abstractmethod
 48    def build_base_prompt(self) -> str:
 49        """Build and return the complete prompt string.
 50
 51        Returns:
 52            str: The constructed prompt.
 53        """
 54        pass
 55
 56    @classmethod
 57    def prompt_builder_name(cls) -> str:
 58        """Returns the name of the prompt builder, to be used for persisting into the datastore.
 59
 60        Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible.
 61
 62        Returns:
 63            str: The prompt builder name in snake_case format.
 64        """
 65        return snake_case(cls.__name__)
 66
 67    def build_user_message(self, input: Dict | str) -> str:
 68        """Build a user message from the input.
 69
 70        Args:
 71            input (Union[Dict, str]): The input to format into a message.
 72
 73        Returns:
 74            str: The formatted user message.
 75        """
 76        if isinstance(input, Dict):
 77            return f"The input is:\n{json.dumps(input, indent=2, ensure_ascii=False)}"
 78
 79        return f"The input is:\n{input}"
 80
 81    def chain_of_thought_prompt(self) -> str | None:
 82        """Build and return the chain of thought prompt string.
 83
 84        Returns:
 85            str: The constructed chain of thought prompt.
 86        """
 87        return None
 88
 89    def build_prompt_for_ui(self) -> str:
 90        """Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages.
 91
 92        Designed for end-user consumption, not for model consumption.
 93
 94        Returns:
 95            str: The constructed prompt string.
 96        """
 97        base_prompt = self.build_prompt(include_json_instructions=False)
 98        cot_prompt = self.chain_of_thought_prompt()
 99        if cot_prompt:
100            base_prompt += "\n# Thinking Instructions\n\n" + cot_prompt
101        return base_prompt
102
103
104class SimplePromptBuilder(BasePromptBuilder):
105    """A basic prompt builder that combines task instruction with requirements."""
106
107    def build_base_prompt(self) -> str:
108        """Build a simple prompt with instruction and requirements.
109
110        Returns:
111            str: The constructed prompt string.
112        """
113        base_prompt = self.task.instruction
114
115        # TODO: this is just a quick version. Formatting and best practices TBD
116        if len(self.task.requirements) > 0:
117            base_prompt += (
118                "\n\nYour response should respect the following requirements:\n"
119            )
120            # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)...
121            for i, requirement in enumerate(self.task.requirements):
122                base_prompt += f"{i + 1}) {requirement.instruction}\n"
123
124        return base_prompt
125
126
127class MultiShotPromptBuilder(BasePromptBuilder):
128    """A prompt builder that includes multiple examples in the prompt."""
129
130    @classmethod
131    def example_count(cls) -> int:
132        """Get the maximum number of examples to include in the prompt.
133
134        Returns:
135            int: The maximum number of examples (default 25).
136        """
137        return 25
138
139    def build_base_prompt(self) -> str:
140        """Build a prompt with instruction, requirements, and multiple examples.
141
142        Returns:
143            str: The constructed prompt string with examples.
144        """
145        base_prompt = f"# Instruction\n\n{self.task.instruction}\n\n"
146
147        if len(self.task.requirements) > 0:
148            base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n"
149            for i, requirement in enumerate(self.task.requirements):
150                base_prompt += f"{i + 1}) {requirement.instruction}\n"
151            base_prompt += "\n"
152
153        valid_examples = self.collect_examples()
154
155        if len(valid_examples) == 0:
156            return base_prompt
157
158        base_prompt += "# Example Outputs\n\n"
159        for i, example in enumerate(valid_examples):
160            base_prompt += self.prompt_section_for_example(i, example)
161
162        return base_prompt
163
164    def prompt_section_for_example(self, index: int, example: TaskRun) -> str:
165        # Prefer repaired output if it exists, otherwise use the regular output
166        output = example.repaired_output or example.output
167        return f"## Example {index + 1}\n\nInput: {example.input}\nOutput: {output.output}\n\n"
168
169    def collect_examples(self) -> list[TaskRun]:
170        valid_examples: list[TaskRun] = []
171        runs = self.task.runs(readonly=True)
172
173        # first pass, we look for repaired outputs. These are the best examples.
174        for run in runs:
175            if len(valid_examples) >= self.__class__.example_count():
176                break
177            if run.repaired_output is not None:
178                valid_examples.append(run)
179
180        # second pass, we look for high quality outputs (rating based)
181        # Minimum is "high_quality" (4 star in star rating scale), then sort by rating
182        # exclude repaired outputs as they were used above
183        runs_with_rating = [
184            run
185            for run in runs
186            if run.output.rating is not None
187            and run.output.rating.value is not None
188            and run.output.rating.is_high_quality()
189            and run.repaired_output is None
190        ]
191        runs_with_rating.sort(
192            key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True
193        )
194        for run in runs_with_rating:
195            if len(valid_examples) >= self.__class__.example_count():
196                break
197            valid_examples.append(run)
198        return valid_examples
199
200
201class FewShotPromptBuilder(MultiShotPromptBuilder):
202    """A prompt builder that includes a small number of examples in the prompt."""
203
204    @classmethod
205    def example_count(cls) -> int:
206        """Get the maximum number of examples to include in the prompt.
207
208        Returns:
209            int: The maximum number of examples (4).
210        """
211        return 4
212
213
214class RepairsPromptBuilder(MultiShotPromptBuilder):
215    """A prompt builder that includes multiple examples in the prompt, including repaired instructions describing what was wrong, and how it was fixed."""
216
217    def prompt_section_for_example(self, index: int, example: TaskRun) -> str:
218        if (
219            not example.repaired_output
220            or not example.repair_instructions
221            or not example.repaired_output.output
222        ):
223            return super().prompt_section_for_example(index, example)
224
225        prompt_section = f"## Example {index + 1}\n\nInput: {example.input}\n\n"
226        prompt_section += (
227            f"Initial Output Which Was Insufficient: {example.output.output}\n\n"
228        )
229        prompt_section += f"Instructions On How to Improve the Initial Output: {example.repair_instructions}\n\n"
230        prompt_section += (
231            f"Repaired Output Which is Sufficient: {example.repaired_output.output}\n\n"
232        )
233        return prompt_section
234
235
236def chain_of_thought_prompt(task: Task) -> str:
237    """Standard implementation to build and return the chain of thought prompt string.
238
239    Returns:
240        str: The constructed chain of thought prompt.
241    """
242
243    cot_instruction = task.thinking_instruction
244    if not cot_instruction:
245        cot_instruction = "Think step by step, explaining your reasoning."
246
247    return cot_instruction
248
249
250class SimpleChainOfThoughtPromptBuilder(SimplePromptBuilder):
251    """A prompt builder that includes a chain of thought prompt on top of the simple prompt."""
252
253    def chain_of_thought_prompt(self) -> str | None:
254        return chain_of_thought_prompt(self.task)
255
256
257class FewShotChainOfThoughtPromptBuilder(FewShotPromptBuilder):
258    """A prompt builder that includes a chain of thought prompt on top of the few shot prompt."""
259
260    def chain_of_thought_prompt(self) -> str | None:
261        return chain_of_thought_prompt(self.task)
262
263
264class MultiShotChainOfThoughtPromptBuilder(MultiShotPromptBuilder):
265    """A prompt builder that includes a chain of thought prompt on top of the multi shot prompt."""
266
267    def chain_of_thought_prompt(self) -> str | None:
268        return chain_of_thought_prompt(self.task)
269
270
271class SavedPromptBuilder(BasePromptBuilder):
272    """A prompt builder that looks up a static prompt."""
273
274    def __init__(self, task: Task, prompt_id: str):
275        super().__init__(task)
276        prompt_model = next(
277            (
278                prompt
279                for prompt in task.prompts(readonly=True)
280                if prompt.id == prompt_id
281            ),
282            None,
283        )
284        if not prompt_model:
285            raise ValueError(f"Prompt ID not found: {prompt_id}")
286        self.prompt_model = prompt_model
287
288    def prompt_id(self) -> str | None:
289        return self.prompt_model.id
290
291    def build_base_prompt(self) -> str:
292        """Returns a saved prompt.
293
294        Returns:
295            str: The prompt string.
296        """
297        return self.prompt_model.prompt
298
299    def chain_of_thought_prompt(self) -> str | None:
300        return self.prompt_model.chain_of_thought_instructions
301
302
303class FineTunePromptBuilder(BasePromptBuilder):
304    """A prompt builder that looks up a fine-tune prompt."""
305
306    def __init__(self, task: Task, nested_fine_tune_id: str):
307        super().__init__(task)
308
309        # IDs are in project_id::task_id::fine_tune_id format
310        self.full_fine_tune_id = nested_fine_tune_id
311        parts = nested_fine_tune_id.split("::")
312        if len(parts) != 3:
313            raise ValueError(
314                f"Invalid fine-tune ID format. Expected 'project_id::task_id::fine_tune_id', got: {nested_fine_tune_id}"
315            )
316        fine_tune_id = parts[2]
317
318        fine_tune_model = next(
319            (
320                fine_tune
321                for fine_tune in task.finetunes(readonly=True)
322                if fine_tune.id == fine_tune_id
323            ),
324            None,
325        )
326        if not fine_tune_model:
327            raise ValueError(f"Fine-tune ID not found: {fine_tune_id}")
328        self.fine_tune_model = fine_tune_model
329
330    def prompt_id(self) -> str | None:
331        return self.full_fine_tune_id
332
333    def build_base_prompt(self) -> str:
334        return self.fine_tune_model.system_message
335
336    def chain_of_thought_prompt(self) -> str | None:
337        return self.fine_tune_model.thinking_instructions
338
339
340# TODO P2: we end up with 2 IDs for these: the keys here (ui_name) and the prompt_builder_name from the class
341# We end up maintaining this in _prompt_generators as well.
342prompt_builder_registry = {
343    "simple_prompt_builder": SimplePromptBuilder,
344    "multi_shot_prompt_builder": MultiShotPromptBuilder,
345    "few_shot_prompt_builder": FewShotPromptBuilder,
346    "repairs_prompt_builder": RepairsPromptBuilder,
347    "simple_chain_of_thought_prompt_builder": SimpleChainOfThoughtPromptBuilder,
348    "few_shot_chain_of_thought_prompt_builder": FewShotChainOfThoughtPromptBuilder,
349    "multi_shot_chain_of_thought_prompt_builder": MultiShotChainOfThoughtPromptBuilder,
350}
351
352
353# Our UI has some names that are not the same as the class names, which also hint parameters.
354def prompt_builder_from_ui_name(ui_name: str, task: Task) -> BasePromptBuilder:
355    """Convert a name used in the UI to the corresponding prompt builder class.
356
357    Args:
358        ui_name (str): The UI name for the prompt builder type.
359
360    Returns:
361        type[BasePromptBuilder]: The corresponding prompt builder class.
362
363    Raises:
364        ValueError: If the UI name is not recognized.
365    """
366
367    # Saved prompts are prefixed with "id::"
368    if ui_name.startswith("id::"):
369        prompt_id = ui_name[4:]
370        return SavedPromptBuilder(task, prompt_id)
371
372    # Fine-tune prompts are prefixed with "fine_tune_prompt::"
373    if ui_name.startswith("fine_tune_prompt::"):
374        fine_tune_id = ui_name[18:]
375        return FineTunePromptBuilder(task, fine_tune_id)
376
377    match ui_name:
378        case "basic":
379            return SimplePromptBuilder(task)
380        case "few_shot":
381            return FewShotPromptBuilder(task)
382        case "many_shot":
383            return MultiShotPromptBuilder(task)
384        case "repairs":
385            return RepairsPromptBuilder(task)
386        case "simple_chain_of_thought":
387            return SimpleChainOfThoughtPromptBuilder(task)
388        case "few_shot_chain_of_thought":
389            return FewShotChainOfThoughtPromptBuilder(task)
390        case "multi_shot_chain_of_thought":
391            return MultiShotChainOfThoughtPromptBuilder(task)
392        case _:
393            raise ValueError(f"Unknown prompt builder: {ui_name}")
class BasePromptBuilder:
 10class BasePromptBuilder(metaclass=ABCMeta):
 11    """Base class for building prompts from tasks.
 12
 13    Provides the core interface and basic functionality for prompt builders.
 14    """
 15
 16    def __init__(self, task: Task):
 17        """Initialize the prompt builder with a task.
 18
 19        Args:
 20            task (Task): The task containing instructions and requirements.
 21        """
 22        self.task = task
 23
 24    def prompt_id(self) -> str | None:
 25        """Returns the ID of the prompt, scoped to this builder.
 26
 27        Returns:
 28            str | None: The ID of the prompt, or None if not set.
 29        """
 30        return None
 31
 32    def build_prompt(self, include_json_instructions) -> str:
 33        """Build and return the complete prompt string.
 34
 35        Returns:
 36            str: The constructed prompt.
 37        """
 38        prompt = self.build_base_prompt()
 39
 40        if include_json_instructions and self.task.output_schema():
 41            prompt = (
 42                prompt
 43                + f"\n\n# Format Instructions\n\nReturn a JSON object conforming to the following schema:\n```\n{self.task.output_schema()}\n```"
 44            )
 45
 46        return prompt
 47
 48    @abstractmethod
 49    def build_base_prompt(self) -> str:
 50        """Build and return the complete prompt string.
 51
 52        Returns:
 53            str: The constructed prompt.
 54        """
 55        pass
 56
 57    @classmethod
 58    def prompt_builder_name(cls) -> str:
 59        """Returns the name of the prompt builder, to be used for persisting into the datastore.
 60
 61        Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible.
 62
 63        Returns:
 64            str: The prompt builder name in snake_case format.
 65        """
 66        return snake_case(cls.__name__)
 67
 68    def build_user_message(self, input: Dict | str) -> str:
 69        """Build a user message from the input.
 70
 71        Args:
 72            input (Union[Dict, str]): The input to format into a message.
 73
 74        Returns:
 75            str: The formatted user message.
 76        """
 77        if isinstance(input, Dict):
 78            return f"The input is:\n{json.dumps(input, indent=2, ensure_ascii=False)}"
 79
 80        return f"The input is:\n{input}"
 81
 82    def chain_of_thought_prompt(self) -> str | None:
 83        """Build and return the chain of thought prompt string.
 84
 85        Returns:
 86            str: The constructed chain of thought prompt.
 87        """
 88        return None
 89
 90    def build_prompt_for_ui(self) -> str:
 91        """Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages.
 92
 93        Designed for end-user consumption, not for model consumption.
 94
 95        Returns:
 96            str: The constructed prompt string.
 97        """
 98        base_prompt = self.build_prompt(include_json_instructions=False)
 99        cot_prompt = self.chain_of_thought_prompt()
100        if cot_prompt:
101            base_prompt += "\n# Thinking Instructions\n\n" + cot_prompt
102        return base_prompt

Base class for building prompts from tasks.

Provides the core interface and basic functionality for prompt builders.

BasePromptBuilder(task: kiln_ai.datamodel.Task)
16    def __init__(self, task: Task):
17        """Initialize the prompt builder with a task.
18
19        Args:
20            task (Task): The task containing instructions and requirements.
21        """
22        self.task = task

Initialize the prompt builder with a task.

Args: task (Task): The task containing instructions and requirements.

task
def prompt_id(self) -> str | None:
24    def prompt_id(self) -> str | None:
25        """Returns the ID of the prompt, scoped to this builder.
26
27        Returns:
28            str | None: The ID of the prompt, or None if not set.
29        """
30        return None

Returns the ID of the prompt, scoped to this builder.

Returns: str | None: The ID of the prompt, or None if not set.

def build_prompt(self, include_json_instructions) -> str:
32    def build_prompt(self, include_json_instructions) -> str:
33        """Build and return the complete prompt string.
34
35        Returns:
36            str: The constructed prompt.
37        """
38        prompt = self.build_base_prompt()
39
40        if include_json_instructions and self.task.output_schema():
41            prompt = (
42                prompt
43                + f"\n\n# Format Instructions\n\nReturn a JSON object conforming to the following schema:\n```\n{self.task.output_schema()}\n```"
44            )
45
46        return prompt

Build and return the complete prompt string.

Returns: str: The constructed prompt.

@abstractmethod
def build_base_prompt(self) -> str:
48    @abstractmethod
49    def build_base_prompt(self) -> str:
50        """Build and return the complete prompt string.
51
52        Returns:
53            str: The constructed prompt.
54        """
55        pass

Build and return the complete prompt string.

Returns: str: The constructed prompt.

@classmethod
def prompt_builder_name(cls) -> str:
57    @classmethod
58    def prompt_builder_name(cls) -> str:
59        """Returns the name of the prompt builder, to be used for persisting into the datastore.
60
61        Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible.
62
63        Returns:
64            str: The prompt builder name in snake_case format.
65        """
66        return snake_case(cls.__name__)

Returns the name of the prompt builder, to be used for persisting into the datastore.

Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible.

Returns: str: The prompt builder name in snake_case format.

def build_user_message(self, input: Union[Dict, str]) -> str:
68    def build_user_message(self, input: Dict | str) -> str:
69        """Build a user message from the input.
70
71        Args:
72            input (Union[Dict, str]): The input to format into a message.
73
74        Returns:
75            str: The formatted user message.
76        """
77        if isinstance(input, Dict):
78            return f"The input is:\n{json.dumps(input, indent=2, ensure_ascii=False)}"
79
80        return f"The input is:\n{input}"

Build a user message from the input.

Args: input (Union[Dict, str]): The input to format into a message.

Returns: str: The formatted user message.

def chain_of_thought_prompt(self) -> str | None:
82    def chain_of_thought_prompt(self) -> str | None:
83        """Build and return the chain of thought prompt string.
84
85        Returns:
86            str: The constructed chain of thought prompt.
87        """
88        return None

Build and return the chain of thought prompt string.

Returns: str: The constructed chain of thought prompt.

def build_prompt_for_ui(self) -> str:
 90    def build_prompt_for_ui(self) -> str:
 91        """Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages.
 92
 93        Designed for end-user consumption, not for model consumption.
 94
 95        Returns:
 96            str: The constructed prompt string.
 97        """
 98        base_prompt = self.build_prompt(include_json_instructions=False)
 99        cot_prompt = self.chain_of_thought_prompt()
100        if cot_prompt:
101            base_prompt += "\n# Thinking Instructions\n\n" + cot_prompt
102        return base_prompt

Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages.

Designed for end-user consumption, not for model consumption.

Returns: str: The constructed prompt string.

class SimplePromptBuilder(BasePromptBuilder):
105class SimplePromptBuilder(BasePromptBuilder):
106    """A basic prompt builder that combines task instruction with requirements."""
107
108    def build_base_prompt(self) -> str:
109        """Build a simple prompt with instruction and requirements.
110
111        Returns:
112            str: The constructed prompt string.
113        """
114        base_prompt = self.task.instruction
115
116        # TODO: this is just a quick version. Formatting and best practices TBD
117        if len(self.task.requirements) > 0:
118            base_prompt += (
119                "\n\nYour response should respect the following requirements:\n"
120            )
121            # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)...
122            for i, requirement in enumerate(self.task.requirements):
123                base_prompt += f"{i + 1}) {requirement.instruction}\n"
124
125        return base_prompt

A basic prompt builder that combines task instruction with requirements.

def build_base_prompt(self) -> str:
108    def build_base_prompt(self) -> str:
109        """Build a simple prompt with instruction and requirements.
110
111        Returns:
112            str: The constructed prompt string.
113        """
114        base_prompt = self.task.instruction
115
116        # TODO: this is just a quick version. Formatting and best practices TBD
117        if len(self.task.requirements) > 0:
118            base_prompt += (
119                "\n\nYour response should respect the following requirements:\n"
120            )
121            # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)...
122            for i, requirement in enumerate(self.task.requirements):
123                base_prompt += f"{i + 1}) {requirement.instruction}\n"
124
125        return base_prompt

Build a simple prompt with instruction and requirements.

Returns: str: The constructed prompt string.

class MultiShotPromptBuilder(BasePromptBuilder):
128class MultiShotPromptBuilder(BasePromptBuilder):
129    """A prompt builder that includes multiple examples in the prompt."""
130
131    @classmethod
132    def example_count(cls) -> int:
133        """Get the maximum number of examples to include in the prompt.
134
135        Returns:
136            int: The maximum number of examples (default 25).
137        """
138        return 25
139
140    def build_base_prompt(self) -> str:
141        """Build a prompt with instruction, requirements, and multiple examples.
142
143        Returns:
144            str: The constructed prompt string with examples.
145        """
146        base_prompt = f"# Instruction\n\n{self.task.instruction}\n\n"
147
148        if len(self.task.requirements) > 0:
149            base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n"
150            for i, requirement in enumerate(self.task.requirements):
151                base_prompt += f"{i + 1}) {requirement.instruction}\n"
152            base_prompt += "\n"
153
154        valid_examples = self.collect_examples()
155
156        if len(valid_examples) == 0:
157            return base_prompt
158
159        base_prompt += "# Example Outputs\n\n"
160        for i, example in enumerate(valid_examples):
161            base_prompt += self.prompt_section_for_example(i, example)
162
163        return base_prompt
164
165    def prompt_section_for_example(self, index: int, example: TaskRun) -> str:
166        # Prefer repaired output if it exists, otherwise use the regular output
167        output = example.repaired_output or example.output
168        return f"## Example {index + 1}\n\nInput: {example.input}\nOutput: {output.output}\n\n"
169
170    def collect_examples(self) -> list[TaskRun]:
171        valid_examples: list[TaskRun] = []
172        runs = self.task.runs(readonly=True)
173
174        # first pass, we look for repaired outputs. These are the best examples.
175        for run in runs:
176            if len(valid_examples) >= self.__class__.example_count():
177                break
178            if run.repaired_output is not None:
179                valid_examples.append(run)
180
181        # second pass, we look for high quality outputs (rating based)
182        # Minimum is "high_quality" (4 star in star rating scale), then sort by rating
183        # exclude repaired outputs as they were used above
184        runs_with_rating = [
185            run
186            for run in runs
187            if run.output.rating is not None
188            and run.output.rating.value is not None
189            and run.output.rating.is_high_quality()
190            and run.repaired_output is None
191        ]
192        runs_with_rating.sort(
193            key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True
194        )
195        for run in runs_with_rating:
196            if len(valid_examples) >= self.__class__.example_count():
197                break
198            valid_examples.append(run)
199        return valid_examples

A prompt builder that includes multiple examples in the prompt.

@classmethod
def example_count(cls) -> int:
131    @classmethod
132    def example_count(cls) -> int:
133        """Get the maximum number of examples to include in the prompt.
134
135        Returns:
136            int: The maximum number of examples (default 25).
137        """
138        return 25

Get the maximum number of examples to include in the prompt.

Returns: int: The maximum number of examples (default 25).

def build_base_prompt(self) -> str:
140    def build_base_prompt(self) -> str:
141        """Build a prompt with instruction, requirements, and multiple examples.
142
143        Returns:
144            str: The constructed prompt string with examples.
145        """
146        base_prompt = f"# Instruction\n\n{self.task.instruction}\n\n"
147
148        if len(self.task.requirements) > 0:
149            base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n"
150            for i, requirement in enumerate(self.task.requirements):
151                base_prompt += f"{i + 1}) {requirement.instruction}\n"
152            base_prompt += "\n"
153
154        valid_examples = self.collect_examples()
155
156        if len(valid_examples) == 0:
157            return base_prompt
158
159        base_prompt += "# Example Outputs\n\n"
160        for i, example in enumerate(valid_examples):
161            base_prompt += self.prompt_section_for_example(i, example)
162
163        return base_prompt

Build a prompt with instruction, requirements, and multiple examples.

Returns: str: The constructed prompt string with examples.

def prompt_section_for_example(self, index: int, example: kiln_ai.datamodel.TaskRun) -> str:
165    def prompt_section_for_example(self, index: int, example: TaskRun) -> str:
166        # Prefer repaired output if it exists, otherwise use the regular output
167        output = example.repaired_output or example.output
168        return f"## Example {index + 1}\n\nInput: {example.input}\nOutput: {output.output}\n\n"
def collect_examples(self) -> list[kiln_ai.datamodel.TaskRun]:
170    def collect_examples(self) -> list[TaskRun]:
171        valid_examples: list[TaskRun] = []
172        runs = self.task.runs(readonly=True)
173
174        # first pass, we look for repaired outputs. These are the best examples.
175        for run in runs:
176            if len(valid_examples) >= self.__class__.example_count():
177                break
178            if run.repaired_output is not None:
179                valid_examples.append(run)
180
181        # second pass, we look for high quality outputs (rating based)
182        # Minimum is "high_quality" (4 star in star rating scale), then sort by rating
183        # exclude repaired outputs as they were used above
184        runs_with_rating = [
185            run
186            for run in runs
187            if run.output.rating is not None
188            and run.output.rating.value is not None
189            and run.output.rating.is_high_quality()
190            and run.repaired_output is None
191        ]
192        runs_with_rating.sort(
193            key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True
194        )
195        for run in runs_with_rating:
196            if len(valid_examples) >= self.__class__.example_count():
197                break
198            valid_examples.append(run)
199        return valid_examples
class FewShotPromptBuilder(MultiShotPromptBuilder):
202class FewShotPromptBuilder(MultiShotPromptBuilder):
203    """A prompt builder that includes a small number of examples in the prompt."""
204
205    @classmethod
206    def example_count(cls) -> int:
207        """Get the maximum number of examples to include in the prompt.
208
209        Returns:
210            int: The maximum number of examples (4).
211        """
212        return 4

A prompt builder that includes a small number of examples in the prompt.

@classmethod
def example_count(cls) -> int:
205    @classmethod
206    def example_count(cls) -> int:
207        """Get the maximum number of examples to include in the prompt.
208
209        Returns:
210            int: The maximum number of examples (4).
211        """
212        return 4

Get the maximum number of examples to include in the prompt.

Returns: int: The maximum number of examples (4).

class RepairsPromptBuilder(MultiShotPromptBuilder):
215class RepairsPromptBuilder(MultiShotPromptBuilder):
216    """A prompt builder that includes multiple examples in the prompt, including repaired instructions describing what was wrong, and how it was fixed."""
217
218    def prompt_section_for_example(self, index: int, example: TaskRun) -> str:
219        if (
220            not example.repaired_output
221            or not example.repair_instructions
222            or not example.repaired_output.output
223        ):
224            return super().prompt_section_for_example(index, example)
225
226        prompt_section = f"## Example {index + 1}\n\nInput: {example.input}\n\n"
227        prompt_section += (
228            f"Initial Output Which Was Insufficient: {example.output.output}\n\n"
229        )
230        prompt_section += f"Instructions On How to Improve the Initial Output: {example.repair_instructions}\n\n"
231        prompt_section += (
232            f"Repaired Output Which is Sufficient: {example.repaired_output.output}\n\n"
233        )
234        return prompt_section

A prompt builder that includes multiple examples in the prompt, including repaired instructions describing what was wrong, and how it was fixed.

def prompt_section_for_example(self, index: int, example: kiln_ai.datamodel.TaskRun) -> str:
218    def prompt_section_for_example(self, index: int, example: TaskRun) -> str:
219        if (
220            not example.repaired_output
221            or not example.repair_instructions
222            or not example.repaired_output.output
223        ):
224            return super().prompt_section_for_example(index, example)
225
226        prompt_section = f"## Example {index + 1}\n\nInput: {example.input}\n\n"
227        prompt_section += (
228            f"Initial Output Which Was Insufficient: {example.output.output}\n\n"
229        )
230        prompt_section += f"Instructions On How to Improve the Initial Output: {example.repair_instructions}\n\n"
231        prompt_section += (
232            f"Repaired Output Which is Sufficient: {example.repaired_output.output}\n\n"
233        )
234        return prompt_section
def chain_of_thought_prompt(task: kiln_ai.datamodel.Task) -> str:
237def chain_of_thought_prompt(task: Task) -> str:
238    """Standard implementation to build and return the chain of thought prompt string.
239
240    Returns:
241        str: The constructed chain of thought prompt.
242    """
243
244    cot_instruction = task.thinking_instruction
245    if not cot_instruction:
246        cot_instruction = "Think step by step, explaining your reasoning."
247
248    return cot_instruction

Standard implementation to build and return the chain of thought prompt string.

Returns: str: The constructed chain of thought prompt.

class SimpleChainOfThoughtPromptBuilder(SimplePromptBuilder):
251class SimpleChainOfThoughtPromptBuilder(SimplePromptBuilder):
252    """A prompt builder that includes a chain of thought prompt on top of the simple prompt."""
253
254    def chain_of_thought_prompt(self) -> str | None:
255        return chain_of_thought_prompt(self.task)

A prompt builder that includes a chain of thought prompt on top of the simple prompt.

def chain_of_thought_prompt(self) -> str | None:
254    def chain_of_thought_prompt(self) -> str | None:
255        return chain_of_thought_prompt(self.task)

Build and return the chain of thought prompt string.

Returns: str: The constructed chain of thought prompt.

class FewShotChainOfThoughtPromptBuilder(FewShotPromptBuilder):
258class FewShotChainOfThoughtPromptBuilder(FewShotPromptBuilder):
259    """A prompt builder that includes a chain of thought prompt on top of the few shot prompt."""
260
261    def chain_of_thought_prompt(self) -> str | None:
262        return chain_of_thought_prompt(self.task)

A prompt builder that includes a chain of thought prompt on top of the few shot prompt.

def chain_of_thought_prompt(self) -> str | None:
261    def chain_of_thought_prompt(self) -> str | None:
262        return chain_of_thought_prompt(self.task)

Build and return the chain of thought prompt string.

Returns: str: The constructed chain of thought prompt.

class MultiShotChainOfThoughtPromptBuilder(MultiShotPromptBuilder):
265class MultiShotChainOfThoughtPromptBuilder(MultiShotPromptBuilder):
266    """A prompt builder that includes a chain of thought prompt on top of the multi shot prompt."""
267
268    def chain_of_thought_prompt(self) -> str | None:
269        return chain_of_thought_prompt(self.task)

A prompt builder that includes a chain of thought prompt on top of the multi shot prompt.

def chain_of_thought_prompt(self) -> str | None:
268    def chain_of_thought_prompt(self) -> str | None:
269        return chain_of_thought_prompt(self.task)

Build and return the chain of thought prompt string.

Returns: str: The constructed chain of thought prompt.

class SavedPromptBuilder(BasePromptBuilder):
272class SavedPromptBuilder(BasePromptBuilder):
273    """A prompt builder that looks up a static prompt."""
274
275    def __init__(self, task: Task, prompt_id: str):
276        super().__init__(task)
277        prompt_model = next(
278            (
279                prompt
280                for prompt in task.prompts(readonly=True)
281                if prompt.id == prompt_id
282            ),
283            None,
284        )
285        if not prompt_model:
286            raise ValueError(f"Prompt ID not found: {prompt_id}")
287        self.prompt_model = prompt_model
288
289    def prompt_id(self) -> str | None:
290        return self.prompt_model.id
291
292    def build_base_prompt(self) -> str:
293        """Returns a saved prompt.
294
295        Returns:
296            str: The prompt string.
297        """
298        return self.prompt_model.prompt
299
300    def chain_of_thought_prompt(self) -> str | None:
301        return self.prompt_model.chain_of_thought_instructions

A prompt builder that looks up a static prompt.

SavedPromptBuilder(task: kiln_ai.datamodel.Task, prompt_id: str)
275    def __init__(self, task: Task, prompt_id: str):
276        super().__init__(task)
277        prompt_model = next(
278            (
279                prompt
280                for prompt in task.prompts(readonly=True)
281                if prompt.id == prompt_id
282            ),
283            None,
284        )
285        if not prompt_model:
286            raise ValueError(f"Prompt ID not found: {prompt_id}")
287        self.prompt_model = prompt_model

Initialize the prompt builder with a task.

Args: task (Task): The task containing instructions and requirements.

prompt_model
def prompt_id(self) -> str | None:
289    def prompt_id(self) -> str | None:
290        return self.prompt_model.id

Returns the ID of the prompt, scoped to this builder.

Returns: str | None: The ID of the prompt, or None if not set.

def build_base_prompt(self) -> str:
292    def build_base_prompt(self) -> str:
293        """Returns a saved prompt.
294
295        Returns:
296            str: The prompt string.
297        """
298        return self.prompt_model.prompt

Returns a saved prompt.

Returns: str: The prompt string.

def chain_of_thought_prompt(self) -> str | None:
300    def chain_of_thought_prompt(self) -> str | None:
301        return self.prompt_model.chain_of_thought_instructions

Build and return the chain of thought prompt string.

Returns: str: The constructed chain of thought prompt.

class FineTunePromptBuilder(BasePromptBuilder):
304class FineTunePromptBuilder(BasePromptBuilder):
305    """A prompt builder that looks up a fine-tune prompt."""
306
307    def __init__(self, task: Task, nested_fine_tune_id: str):
308        super().__init__(task)
309
310        # IDs are in project_id::task_id::fine_tune_id format
311        self.full_fine_tune_id = nested_fine_tune_id
312        parts = nested_fine_tune_id.split("::")
313        if len(parts) != 3:
314            raise ValueError(
315                f"Invalid fine-tune ID format. Expected 'project_id::task_id::fine_tune_id', got: {nested_fine_tune_id}"
316            )
317        fine_tune_id = parts[2]
318
319        fine_tune_model = next(
320            (
321                fine_tune
322                for fine_tune in task.finetunes(readonly=True)
323                if fine_tune.id == fine_tune_id
324            ),
325            None,
326        )
327        if not fine_tune_model:
328            raise ValueError(f"Fine-tune ID not found: {fine_tune_id}")
329        self.fine_tune_model = fine_tune_model
330
331    def prompt_id(self) -> str | None:
332        return self.full_fine_tune_id
333
334    def build_base_prompt(self) -> str:
335        return self.fine_tune_model.system_message
336
337    def chain_of_thought_prompt(self) -> str | None:
338        return self.fine_tune_model.thinking_instructions

A prompt builder that looks up a fine-tune prompt.

FineTunePromptBuilder(task: kiln_ai.datamodel.Task, nested_fine_tune_id: str)
307    def __init__(self, task: Task, nested_fine_tune_id: str):
308        super().__init__(task)
309
310        # IDs are in project_id::task_id::fine_tune_id format
311        self.full_fine_tune_id = nested_fine_tune_id
312        parts = nested_fine_tune_id.split("::")
313        if len(parts) != 3:
314            raise ValueError(
315                f"Invalid fine-tune ID format. Expected 'project_id::task_id::fine_tune_id', got: {nested_fine_tune_id}"
316            )
317        fine_tune_id = parts[2]
318
319        fine_tune_model = next(
320            (
321                fine_tune
322                for fine_tune in task.finetunes(readonly=True)
323                if fine_tune.id == fine_tune_id
324            ),
325            None,
326        )
327        if not fine_tune_model:
328            raise ValueError(f"Fine-tune ID not found: {fine_tune_id}")
329        self.fine_tune_model = fine_tune_model

Initialize the prompt builder with a task.

Args: task (Task): The task containing instructions and requirements.

full_fine_tune_id
fine_tune_model
def prompt_id(self) -> str | None:
331    def prompt_id(self) -> str | None:
332        return self.full_fine_tune_id

Returns the ID of the prompt, scoped to this builder.

Returns: str | None: The ID of the prompt, or None if not set.

def build_base_prompt(self) -> str:
334    def build_base_prompt(self) -> str:
335        return self.fine_tune_model.system_message

Build and return the complete prompt string.

Returns: str: The constructed prompt.

def chain_of_thought_prompt(self) -> str | None:
337    def chain_of_thought_prompt(self) -> str | None:
338        return self.fine_tune_model.thinking_instructions

Build and return the chain of thought prompt string.

Returns: str: The constructed chain of thought prompt.

prompt_builder_registry = {'simple_prompt_builder': <class 'SimplePromptBuilder'>, 'multi_shot_prompt_builder': <class 'MultiShotPromptBuilder'>, 'few_shot_prompt_builder': <class 'FewShotPromptBuilder'>, 'repairs_prompt_builder': <class 'RepairsPromptBuilder'>, 'simple_chain_of_thought_prompt_builder': <class 'SimpleChainOfThoughtPromptBuilder'>, 'few_shot_chain_of_thought_prompt_builder': <class 'FewShotChainOfThoughtPromptBuilder'>, 'multi_shot_chain_of_thought_prompt_builder': <class 'MultiShotChainOfThoughtPromptBuilder'>}
def prompt_builder_from_ui_name( ui_name: str, task: kiln_ai.datamodel.Task) -> BasePromptBuilder:
355def prompt_builder_from_ui_name(ui_name: str, task: Task) -> BasePromptBuilder:
356    """Convert a name used in the UI to the corresponding prompt builder class.
357
358    Args:
359        ui_name (str): The UI name for the prompt builder type.
360
361    Returns:
362        type[BasePromptBuilder]: The corresponding prompt builder class.
363
364    Raises:
365        ValueError: If the UI name is not recognized.
366    """
367
368    # Saved prompts are prefixed with "id::"
369    if ui_name.startswith("id::"):
370        prompt_id = ui_name[4:]
371        return SavedPromptBuilder(task, prompt_id)
372
373    # Fine-tune prompts are prefixed with "fine_tune_prompt::"
374    if ui_name.startswith("fine_tune_prompt::"):
375        fine_tune_id = ui_name[18:]
376        return FineTunePromptBuilder(task, fine_tune_id)
377
378    match ui_name:
379        case "basic":
380            return SimplePromptBuilder(task)
381        case "few_shot":
382            return FewShotPromptBuilder(task)
383        case "many_shot":
384            return MultiShotPromptBuilder(task)
385        case "repairs":
386            return RepairsPromptBuilder(task)
387        case "simple_chain_of_thought":
388            return SimpleChainOfThoughtPromptBuilder(task)
389        case "few_shot_chain_of_thought":
390            return FewShotChainOfThoughtPromptBuilder(task)
391        case "multi_shot_chain_of_thought":
392            return MultiShotChainOfThoughtPromptBuilder(task)
393        case _:
394            raise ValueError(f"Unknown prompt builder: {ui_name}")

Convert a name used in the UI to the corresponding prompt builder class.

Args: ui_name (str): The UI name for the prompt builder type.

Returns: type[BasePromptBuilder]: The corresponding prompt builder class.

Raises: ValueError: If the UI name is not recognized.