kiln_ai.adapters.prompt_builders
1import json 2from abc import ABCMeta, abstractmethod 3from typing import Dict 4 5from kiln_ai.datamodel import Task, TaskRun 6from kiln_ai.utils.formatting import snake_case 7 8 9class BasePromptBuilder(metaclass=ABCMeta): 10 """Base class for building prompts from tasks. 11 12 Provides the core interface and basic functionality for prompt builders. 13 """ 14 15 def __init__(self, task: Task): 16 """Initialize the prompt builder with a task. 17 18 Args: 19 task (Task): The task containing instructions and requirements. 20 """ 21 self.task = task 22 23 def prompt_id(self) -> str | None: 24 """Returns the ID of the prompt, scoped to this builder. 25 26 Returns: 27 str | None: The ID of the prompt, or None if not set. 28 """ 29 return None 30 31 def build_prompt(self, include_json_instructions) -> str: 32 """Build and return the complete prompt string. 33 34 Returns: 35 str: The constructed prompt. 36 """ 37 prompt = self.build_base_prompt() 38 39 if include_json_instructions and self.task.output_schema(): 40 prompt = ( 41 prompt 42 + f"\n\n# Format Instructions\n\nReturn a JSON object conforming to the following schema:\n```\n{self.task.output_schema()}\n```" 43 ) 44 45 return prompt 46 47 @abstractmethod 48 def build_base_prompt(self) -> str: 49 """Build and return the complete prompt string. 50 51 Returns: 52 str: The constructed prompt. 53 """ 54 pass 55 56 @classmethod 57 def prompt_builder_name(cls) -> str: 58 """Returns the name of the prompt builder, to be used for persisting into the datastore. 59 60 Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible. 61 62 Returns: 63 str: The prompt builder name in snake_case format. 64 """ 65 return snake_case(cls.__name__) 66 67 def build_user_message(self, input: Dict | str) -> str: 68 """Build a user message from the input. 69 70 Args: 71 input (Union[Dict, str]): The input to format into a message. 72 73 Returns: 74 str: The formatted user message. 75 """ 76 if isinstance(input, Dict): 77 return f"The input is:\n{json.dumps(input, indent=2, ensure_ascii=False)}" 78 79 return f"The input is:\n{input}" 80 81 def chain_of_thought_prompt(self) -> str | None: 82 """Build and return the chain of thought prompt string. 83 84 Returns: 85 str: The constructed chain of thought prompt. 86 """ 87 return None 88 89 def build_prompt_for_ui(self) -> str: 90 """Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages. 91 92 Designed for end-user consumption, not for model consumption. 93 94 Returns: 95 str: The constructed prompt string. 96 """ 97 base_prompt = self.build_prompt(include_json_instructions=False) 98 cot_prompt = self.chain_of_thought_prompt() 99 if cot_prompt: 100 base_prompt += "\n# Thinking Instructions\n\n" + cot_prompt 101 return base_prompt 102 103 104class SimplePromptBuilder(BasePromptBuilder): 105 """A basic prompt builder that combines task instruction with requirements.""" 106 107 def build_base_prompt(self) -> str: 108 """Build a simple prompt with instruction and requirements. 109 110 Returns: 111 str: The constructed prompt string. 112 """ 113 base_prompt = self.task.instruction 114 115 # TODO: this is just a quick version. Formatting and best practices TBD 116 if len(self.task.requirements) > 0: 117 base_prompt += ( 118 "\n\nYour response should respect the following requirements:\n" 119 ) 120 # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)... 121 for i, requirement in enumerate(self.task.requirements): 122 base_prompt += f"{i + 1}) {requirement.instruction}\n" 123 124 return base_prompt 125 126 127class MultiShotPromptBuilder(BasePromptBuilder): 128 """A prompt builder that includes multiple examples in the prompt.""" 129 130 @classmethod 131 def example_count(cls) -> int: 132 """Get the maximum number of examples to include in the prompt. 133 134 Returns: 135 int: The maximum number of examples (default 25). 136 """ 137 return 25 138 139 def build_base_prompt(self) -> str: 140 """Build a prompt with instruction, requirements, and multiple examples. 141 142 Returns: 143 str: The constructed prompt string with examples. 144 """ 145 base_prompt = f"# Instruction\n\n{self.task.instruction}\n\n" 146 147 if len(self.task.requirements) > 0: 148 base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n" 149 for i, requirement in enumerate(self.task.requirements): 150 base_prompt += f"{i + 1}) {requirement.instruction}\n" 151 base_prompt += "\n" 152 153 valid_examples = self.collect_examples() 154 155 if len(valid_examples) == 0: 156 return base_prompt 157 158 base_prompt += "# Example Outputs\n\n" 159 for i, example in enumerate(valid_examples): 160 base_prompt += self.prompt_section_for_example(i, example) 161 162 return base_prompt 163 164 def prompt_section_for_example(self, index: int, example: TaskRun) -> str: 165 # Prefer repaired output if it exists, otherwise use the regular output 166 output = example.repaired_output or example.output 167 return f"## Example {index + 1}\n\nInput: {example.input}\nOutput: {output.output}\n\n" 168 169 def collect_examples(self) -> list[TaskRun]: 170 valid_examples: list[TaskRun] = [] 171 runs = self.task.runs(readonly=True) 172 173 # first pass, we look for repaired outputs. These are the best examples. 174 for run in runs: 175 if len(valid_examples) >= self.__class__.example_count(): 176 break 177 if run.repaired_output is not None: 178 valid_examples.append(run) 179 180 # second pass, we look for high quality outputs (rating based) 181 # Minimum is "high_quality" (4 star in star rating scale), then sort by rating 182 # exclude repaired outputs as they were used above 183 runs_with_rating = [ 184 run 185 for run in runs 186 if run.output.rating is not None 187 and run.output.rating.value is not None 188 and run.output.rating.is_high_quality() 189 and run.repaired_output is None 190 ] 191 runs_with_rating.sort( 192 key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True 193 ) 194 for run in runs_with_rating: 195 if len(valid_examples) >= self.__class__.example_count(): 196 break 197 valid_examples.append(run) 198 return valid_examples 199 200 201class FewShotPromptBuilder(MultiShotPromptBuilder): 202 """A prompt builder that includes a small number of examples in the prompt.""" 203 204 @classmethod 205 def example_count(cls) -> int: 206 """Get the maximum number of examples to include in the prompt. 207 208 Returns: 209 int: The maximum number of examples (4). 210 """ 211 return 4 212 213 214class RepairsPromptBuilder(MultiShotPromptBuilder): 215 """A prompt builder that includes multiple examples in the prompt, including repaired instructions describing what was wrong, and how it was fixed.""" 216 217 def prompt_section_for_example(self, index: int, example: TaskRun) -> str: 218 if ( 219 not example.repaired_output 220 or not example.repair_instructions 221 or not example.repaired_output.output 222 ): 223 return super().prompt_section_for_example(index, example) 224 225 prompt_section = f"## Example {index + 1}\n\nInput: {example.input}\n\n" 226 prompt_section += ( 227 f"Initial Output Which Was Insufficient: {example.output.output}\n\n" 228 ) 229 prompt_section += f"Instructions On How to Improve the Initial Output: {example.repair_instructions}\n\n" 230 prompt_section += ( 231 f"Repaired Output Which is Sufficient: {example.repaired_output.output}\n\n" 232 ) 233 return prompt_section 234 235 236def chain_of_thought_prompt(task: Task) -> str: 237 """Standard implementation to build and return the chain of thought prompt string. 238 239 Returns: 240 str: The constructed chain of thought prompt. 241 """ 242 243 cot_instruction = task.thinking_instruction 244 if not cot_instruction: 245 cot_instruction = "Think step by step, explaining your reasoning." 246 247 return cot_instruction 248 249 250class SimpleChainOfThoughtPromptBuilder(SimplePromptBuilder): 251 """A prompt builder that includes a chain of thought prompt on top of the simple prompt.""" 252 253 def chain_of_thought_prompt(self) -> str | None: 254 return chain_of_thought_prompt(self.task) 255 256 257class FewShotChainOfThoughtPromptBuilder(FewShotPromptBuilder): 258 """A prompt builder that includes a chain of thought prompt on top of the few shot prompt.""" 259 260 def chain_of_thought_prompt(self) -> str | None: 261 return chain_of_thought_prompt(self.task) 262 263 264class MultiShotChainOfThoughtPromptBuilder(MultiShotPromptBuilder): 265 """A prompt builder that includes a chain of thought prompt on top of the multi shot prompt.""" 266 267 def chain_of_thought_prompt(self) -> str | None: 268 return chain_of_thought_prompt(self.task) 269 270 271class SavedPromptBuilder(BasePromptBuilder): 272 """A prompt builder that looks up a static prompt.""" 273 274 def __init__(self, task: Task, prompt_id: str): 275 super().__init__(task) 276 prompt_model = next( 277 ( 278 prompt 279 for prompt in task.prompts(readonly=True) 280 if prompt.id == prompt_id 281 ), 282 None, 283 ) 284 if not prompt_model: 285 raise ValueError(f"Prompt ID not found: {prompt_id}") 286 self.prompt_model = prompt_model 287 288 def prompt_id(self) -> str | None: 289 return self.prompt_model.id 290 291 def build_base_prompt(self) -> str: 292 """Returns a saved prompt. 293 294 Returns: 295 str: The prompt string. 296 """ 297 return self.prompt_model.prompt 298 299 def chain_of_thought_prompt(self) -> str | None: 300 return self.prompt_model.chain_of_thought_instructions 301 302 303class FineTunePromptBuilder(BasePromptBuilder): 304 """A prompt builder that looks up a fine-tune prompt.""" 305 306 def __init__(self, task: Task, nested_fine_tune_id: str): 307 super().__init__(task) 308 309 # IDs are in project_id::task_id::fine_tune_id format 310 self.full_fine_tune_id = nested_fine_tune_id 311 parts = nested_fine_tune_id.split("::") 312 if len(parts) != 3: 313 raise ValueError( 314 f"Invalid fine-tune ID format. Expected 'project_id::task_id::fine_tune_id', got: {nested_fine_tune_id}" 315 ) 316 fine_tune_id = parts[2] 317 318 fine_tune_model = next( 319 ( 320 fine_tune 321 for fine_tune in task.finetunes(readonly=True) 322 if fine_tune.id == fine_tune_id 323 ), 324 None, 325 ) 326 if not fine_tune_model: 327 raise ValueError(f"Fine-tune ID not found: {fine_tune_id}") 328 self.fine_tune_model = fine_tune_model 329 330 def prompt_id(self) -> str | None: 331 return self.full_fine_tune_id 332 333 def build_base_prompt(self) -> str: 334 return self.fine_tune_model.system_message 335 336 def chain_of_thought_prompt(self) -> str | None: 337 return self.fine_tune_model.thinking_instructions 338 339 340# TODO P2: we end up with 2 IDs for these: the keys here (ui_name) and the prompt_builder_name from the class 341# We end up maintaining this in _prompt_generators as well. 342prompt_builder_registry = { 343 "simple_prompt_builder": SimplePromptBuilder, 344 "multi_shot_prompt_builder": MultiShotPromptBuilder, 345 "few_shot_prompt_builder": FewShotPromptBuilder, 346 "repairs_prompt_builder": RepairsPromptBuilder, 347 "simple_chain_of_thought_prompt_builder": SimpleChainOfThoughtPromptBuilder, 348 "few_shot_chain_of_thought_prompt_builder": FewShotChainOfThoughtPromptBuilder, 349 "multi_shot_chain_of_thought_prompt_builder": MultiShotChainOfThoughtPromptBuilder, 350} 351 352 353# Our UI has some names that are not the same as the class names, which also hint parameters. 354def prompt_builder_from_ui_name(ui_name: str, task: Task) -> BasePromptBuilder: 355 """Convert a name used in the UI to the corresponding prompt builder class. 356 357 Args: 358 ui_name (str): The UI name for the prompt builder type. 359 360 Returns: 361 type[BasePromptBuilder]: The corresponding prompt builder class. 362 363 Raises: 364 ValueError: If the UI name is not recognized. 365 """ 366 367 # Saved prompts are prefixed with "id::" 368 if ui_name.startswith("id::"): 369 prompt_id = ui_name[4:] 370 return SavedPromptBuilder(task, prompt_id) 371 372 # Fine-tune prompts are prefixed with "fine_tune_prompt::" 373 if ui_name.startswith("fine_tune_prompt::"): 374 fine_tune_id = ui_name[18:] 375 return FineTunePromptBuilder(task, fine_tune_id) 376 377 match ui_name: 378 case "basic": 379 return SimplePromptBuilder(task) 380 case "few_shot": 381 return FewShotPromptBuilder(task) 382 case "many_shot": 383 return MultiShotPromptBuilder(task) 384 case "repairs": 385 return RepairsPromptBuilder(task) 386 case "simple_chain_of_thought": 387 return SimpleChainOfThoughtPromptBuilder(task) 388 case "few_shot_chain_of_thought": 389 return FewShotChainOfThoughtPromptBuilder(task) 390 case "multi_shot_chain_of_thought": 391 return MultiShotChainOfThoughtPromptBuilder(task) 392 case _: 393 raise ValueError(f"Unknown prompt builder: {ui_name}")
10class BasePromptBuilder(metaclass=ABCMeta): 11 """Base class for building prompts from tasks. 12 13 Provides the core interface and basic functionality for prompt builders. 14 """ 15 16 def __init__(self, task: Task): 17 """Initialize the prompt builder with a task. 18 19 Args: 20 task (Task): The task containing instructions and requirements. 21 """ 22 self.task = task 23 24 def prompt_id(self) -> str | None: 25 """Returns the ID of the prompt, scoped to this builder. 26 27 Returns: 28 str | None: The ID of the prompt, or None if not set. 29 """ 30 return None 31 32 def build_prompt(self, include_json_instructions) -> str: 33 """Build and return the complete prompt string. 34 35 Returns: 36 str: The constructed prompt. 37 """ 38 prompt = self.build_base_prompt() 39 40 if include_json_instructions and self.task.output_schema(): 41 prompt = ( 42 prompt 43 + f"\n\n# Format Instructions\n\nReturn a JSON object conforming to the following schema:\n```\n{self.task.output_schema()}\n```" 44 ) 45 46 return prompt 47 48 @abstractmethod 49 def build_base_prompt(self) -> str: 50 """Build and return the complete prompt string. 51 52 Returns: 53 str: The constructed prompt. 54 """ 55 pass 56 57 @classmethod 58 def prompt_builder_name(cls) -> str: 59 """Returns the name of the prompt builder, to be used for persisting into the datastore. 60 61 Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible. 62 63 Returns: 64 str: The prompt builder name in snake_case format. 65 """ 66 return snake_case(cls.__name__) 67 68 def build_user_message(self, input: Dict | str) -> str: 69 """Build a user message from the input. 70 71 Args: 72 input (Union[Dict, str]): The input to format into a message. 73 74 Returns: 75 str: The formatted user message. 76 """ 77 if isinstance(input, Dict): 78 return f"The input is:\n{json.dumps(input, indent=2, ensure_ascii=False)}" 79 80 return f"The input is:\n{input}" 81 82 def chain_of_thought_prompt(self) -> str | None: 83 """Build and return the chain of thought prompt string. 84 85 Returns: 86 str: The constructed chain of thought prompt. 87 """ 88 return None 89 90 def build_prompt_for_ui(self) -> str: 91 """Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages. 92 93 Designed for end-user consumption, not for model consumption. 94 95 Returns: 96 str: The constructed prompt string. 97 """ 98 base_prompt = self.build_prompt(include_json_instructions=False) 99 cot_prompt = self.chain_of_thought_prompt() 100 if cot_prompt: 101 base_prompt += "\n# Thinking Instructions\n\n" + cot_prompt 102 return base_prompt
Base class for building prompts from tasks.
Provides the core interface and basic functionality for prompt builders.
16 def __init__(self, task: Task): 17 """Initialize the prompt builder with a task. 18 19 Args: 20 task (Task): The task containing instructions and requirements. 21 """ 22 self.task = task
Initialize the prompt builder with a task.
Args: task (Task): The task containing instructions and requirements.
24 def prompt_id(self) -> str | None: 25 """Returns the ID of the prompt, scoped to this builder. 26 27 Returns: 28 str | None: The ID of the prompt, or None if not set. 29 """ 30 return None
Returns the ID of the prompt, scoped to this builder.
Returns: str | None: The ID of the prompt, or None if not set.
32 def build_prompt(self, include_json_instructions) -> str: 33 """Build and return the complete prompt string. 34 35 Returns: 36 str: The constructed prompt. 37 """ 38 prompt = self.build_base_prompt() 39 40 if include_json_instructions and self.task.output_schema(): 41 prompt = ( 42 prompt 43 + f"\n\n# Format Instructions\n\nReturn a JSON object conforming to the following schema:\n```\n{self.task.output_schema()}\n```" 44 ) 45 46 return prompt
Build and return the complete prompt string.
Returns: str: The constructed prompt.
48 @abstractmethod 49 def build_base_prompt(self) -> str: 50 """Build and return the complete prompt string. 51 52 Returns: 53 str: The constructed prompt. 54 """ 55 pass
Build and return the complete prompt string.
Returns: str: The constructed prompt.
57 @classmethod 58 def prompt_builder_name(cls) -> str: 59 """Returns the name of the prompt builder, to be used for persisting into the datastore. 60 61 Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible. 62 63 Returns: 64 str: The prompt builder name in snake_case format. 65 """ 66 return snake_case(cls.__name__)
Returns the name of the prompt builder, to be used for persisting into the datastore.
Default implementation gets the name of the prompt builder in snake case. If you change the class name, you should override this so prior saved data is compatible.
Returns: str: The prompt builder name in snake_case format.
68 def build_user_message(self, input: Dict | str) -> str: 69 """Build a user message from the input. 70 71 Args: 72 input (Union[Dict, str]): The input to format into a message. 73 74 Returns: 75 str: The formatted user message. 76 """ 77 if isinstance(input, Dict): 78 return f"The input is:\n{json.dumps(input, indent=2, ensure_ascii=False)}" 79 80 return f"The input is:\n{input}"
Build a user message from the input.
Args: input (Union[Dict, str]): The input to format into a message.
Returns: str: The formatted user message.
82 def chain_of_thought_prompt(self) -> str | None: 83 """Build and return the chain of thought prompt string. 84 85 Returns: 86 str: The constructed chain of thought prompt. 87 """ 88 return None
Build and return the chain of thought prompt string.
Returns: str: The constructed chain of thought prompt.
90 def build_prompt_for_ui(self) -> str: 91 """Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages. 92 93 Designed for end-user consumption, not for model consumption. 94 95 Returns: 96 str: The constructed prompt string. 97 """ 98 base_prompt = self.build_prompt(include_json_instructions=False) 99 cot_prompt = self.chain_of_thought_prompt() 100 if cot_prompt: 101 base_prompt += "\n# Thinking Instructions\n\n" + cot_prompt 102 return base_prompt
Build a prompt for the UI. It includes additional instructions (like chain of thought), even if they are passed to the model in stages.
Designed for end-user consumption, not for model consumption.
Returns: str: The constructed prompt string.
105class SimplePromptBuilder(BasePromptBuilder): 106 """A basic prompt builder that combines task instruction with requirements.""" 107 108 def build_base_prompt(self) -> str: 109 """Build a simple prompt with instruction and requirements. 110 111 Returns: 112 str: The constructed prompt string. 113 """ 114 base_prompt = self.task.instruction 115 116 # TODO: this is just a quick version. Formatting and best practices TBD 117 if len(self.task.requirements) > 0: 118 base_prompt += ( 119 "\n\nYour response should respect the following requirements:\n" 120 ) 121 # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)... 122 for i, requirement in enumerate(self.task.requirements): 123 base_prompt += f"{i + 1}) {requirement.instruction}\n" 124 125 return base_prompt
A basic prompt builder that combines task instruction with requirements.
108 def build_base_prompt(self) -> str: 109 """Build a simple prompt with instruction and requirements. 110 111 Returns: 112 str: The constructed prompt string. 113 """ 114 base_prompt = self.task.instruction 115 116 # TODO: this is just a quick version. Formatting and best practices TBD 117 if len(self.task.requirements) > 0: 118 base_prompt += ( 119 "\n\nYour response should respect the following requirements:\n" 120 ) 121 # iterate requirements, formatting them in numbereed list like 1) task.instruction\n2)... 122 for i, requirement in enumerate(self.task.requirements): 123 base_prompt += f"{i + 1}) {requirement.instruction}\n" 124 125 return base_prompt
Build a simple prompt with instruction and requirements.
Returns: str: The constructed prompt string.
128class MultiShotPromptBuilder(BasePromptBuilder): 129 """A prompt builder that includes multiple examples in the prompt.""" 130 131 @classmethod 132 def example_count(cls) -> int: 133 """Get the maximum number of examples to include in the prompt. 134 135 Returns: 136 int: The maximum number of examples (default 25). 137 """ 138 return 25 139 140 def build_base_prompt(self) -> str: 141 """Build a prompt with instruction, requirements, and multiple examples. 142 143 Returns: 144 str: The constructed prompt string with examples. 145 """ 146 base_prompt = f"# Instruction\n\n{self.task.instruction}\n\n" 147 148 if len(self.task.requirements) > 0: 149 base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n" 150 for i, requirement in enumerate(self.task.requirements): 151 base_prompt += f"{i + 1}) {requirement.instruction}\n" 152 base_prompt += "\n" 153 154 valid_examples = self.collect_examples() 155 156 if len(valid_examples) == 0: 157 return base_prompt 158 159 base_prompt += "# Example Outputs\n\n" 160 for i, example in enumerate(valid_examples): 161 base_prompt += self.prompt_section_for_example(i, example) 162 163 return base_prompt 164 165 def prompt_section_for_example(self, index: int, example: TaskRun) -> str: 166 # Prefer repaired output if it exists, otherwise use the regular output 167 output = example.repaired_output or example.output 168 return f"## Example {index + 1}\n\nInput: {example.input}\nOutput: {output.output}\n\n" 169 170 def collect_examples(self) -> list[TaskRun]: 171 valid_examples: list[TaskRun] = [] 172 runs = self.task.runs(readonly=True) 173 174 # first pass, we look for repaired outputs. These are the best examples. 175 for run in runs: 176 if len(valid_examples) >= self.__class__.example_count(): 177 break 178 if run.repaired_output is not None: 179 valid_examples.append(run) 180 181 # second pass, we look for high quality outputs (rating based) 182 # Minimum is "high_quality" (4 star in star rating scale), then sort by rating 183 # exclude repaired outputs as they were used above 184 runs_with_rating = [ 185 run 186 for run in runs 187 if run.output.rating is not None 188 and run.output.rating.value is not None 189 and run.output.rating.is_high_quality() 190 and run.repaired_output is None 191 ] 192 runs_with_rating.sort( 193 key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True 194 ) 195 for run in runs_with_rating: 196 if len(valid_examples) >= self.__class__.example_count(): 197 break 198 valid_examples.append(run) 199 return valid_examples
A prompt builder that includes multiple examples in the prompt.
131 @classmethod 132 def example_count(cls) -> int: 133 """Get the maximum number of examples to include in the prompt. 134 135 Returns: 136 int: The maximum number of examples (default 25). 137 """ 138 return 25
Get the maximum number of examples to include in the prompt.
Returns: int: The maximum number of examples (default 25).
140 def build_base_prompt(self) -> str: 141 """Build a prompt with instruction, requirements, and multiple examples. 142 143 Returns: 144 str: The constructed prompt string with examples. 145 """ 146 base_prompt = f"# Instruction\n\n{self.task.instruction}\n\n" 147 148 if len(self.task.requirements) > 0: 149 base_prompt += "# Requirements\n\nYour response should respect the following requirements:\n" 150 for i, requirement in enumerate(self.task.requirements): 151 base_prompt += f"{i + 1}) {requirement.instruction}\n" 152 base_prompt += "\n" 153 154 valid_examples = self.collect_examples() 155 156 if len(valid_examples) == 0: 157 return base_prompt 158 159 base_prompt += "# Example Outputs\n\n" 160 for i, example in enumerate(valid_examples): 161 base_prompt += self.prompt_section_for_example(i, example) 162 163 return base_prompt
Build a prompt with instruction, requirements, and multiple examples.
Returns: str: The constructed prompt string with examples.
165 def prompt_section_for_example(self, index: int, example: TaskRun) -> str: 166 # Prefer repaired output if it exists, otherwise use the regular output 167 output = example.repaired_output or example.output 168 return f"## Example {index + 1}\n\nInput: {example.input}\nOutput: {output.output}\n\n"
170 def collect_examples(self) -> list[TaskRun]: 171 valid_examples: list[TaskRun] = [] 172 runs = self.task.runs(readonly=True) 173 174 # first pass, we look for repaired outputs. These are the best examples. 175 for run in runs: 176 if len(valid_examples) >= self.__class__.example_count(): 177 break 178 if run.repaired_output is not None: 179 valid_examples.append(run) 180 181 # second pass, we look for high quality outputs (rating based) 182 # Minimum is "high_quality" (4 star in star rating scale), then sort by rating 183 # exclude repaired outputs as they were used above 184 runs_with_rating = [ 185 run 186 for run in runs 187 if run.output.rating is not None 188 and run.output.rating.value is not None 189 and run.output.rating.is_high_quality() 190 and run.repaired_output is None 191 ] 192 runs_with_rating.sort( 193 key=lambda x: (x.output.rating and x.output.rating.value) or 0, reverse=True 194 ) 195 for run in runs_with_rating: 196 if len(valid_examples) >= self.__class__.example_count(): 197 break 198 valid_examples.append(run) 199 return valid_examples
202class FewShotPromptBuilder(MultiShotPromptBuilder): 203 """A prompt builder that includes a small number of examples in the prompt.""" 204 205 @classmethod 206 def example_count(cls) -> int: 207 """Get the maximum number of examples to include in the prompt. 208 209 Returns: 210 int: The maximum number of examples (4). 211 """ 212 return 4
A prompt builder that includes a small number of examples in the prompt.
205 @classmethod 206 def example_count(cls) -> int: 207 """Get the maximum number of examples to include in the prompt. 208 209 Returns: 210 int: The maximum number of examples (4). 211 """ 212 return 4
Get the maximum number of examples to include in the prompt.
Returns: int: The maximum number of examples (4).
215class RepairsPromptBuilder(MultiShotPromptBuilder): 216 """A prompt builder that includes multiple examples in the prompt, including repaired instructions describing what was wrong, and how it was fixed.""" 217 218 def prompt_section_for_example(self, index: int, example: TaskRun) -> str: 219 if ( 220 not example.repaired_output 221 or not example.repair_instructions 222 or not example.repaired_output.output 223 ): 224 return super().prompt_section_for_example(index, example) 225 226 prompt_section = f"## Example {index + 1}\n\nInput: {example.input}\n\n" 227 prompt_section += ( 228 f"Initial Output Which Was Insufficient: {example.output.output}\n\n" 229 ) 230 prompt_section += f"Instructions On How to Improve the Initial Output: {example.repair_instructions}\n\n" 231 prompt_section += ( 232 f"Repaired Output Which is Sufficient: {example.repaired_output.output}\n\n" 233 ) 234 return prompt_section
A prompt builder that includes multiple examples in the prompt, including repaired instructions describing what was wrong, and how it was fixed.
218 def prompt_section_for_example(self, index: int, example: TaskRun) -> str: 219 if ( 220 not example.repaired_output 221 or not example.repair_instructions 222 or not example.repaired_output.output 223 ): 224 return super().prompt_section_for_example(index, example) 225 226 prompt_section = f"## Example {index + 1}\n\nInput: {example.input}\n\n" 227 prompt_section += ( 228 f"Initial Output Which Was Insufficient: {example.output.output}\n\n" 229 ) 230 prompt_section += f"Instructions On How to Improve the Initial Output: {example.repair_instructions}\n\n" 231 prompt_section += ( 232 f"Repaired Output Which is Sufficient: {example.repaired_output.output}\n\n" 233 ) 234 return prompt_section
237def chain_of_thought_prompt(task: Task) -> str: 238 """Standard implementation to build and return the chain of thought prompt string. 239 240 Returns: 241 str: The constructed chain of thought prompt. 242 """ 243 244 cot_instruction = task.thinking_instruction 245 if not cot_instruction: 246 cot_instruction = "Think step by step, explaining your reasoning." 247 248 return cot_instruction
Standard implementation to build and return the chain of thought prompt string.
Returns: str: The constructed chain of thought prompt.
251class SimpleChainOfThoughtPromptBuilder(SimplePromptBuilder): 252 """A prompt builder that includes a chain of thought prompt on top of the simple prompt.""" 253 254 def chain_of_thought_prompt(self) -> str | None: 255 return chain_of_thought_prompt(self.task)
A prompt builder that includes a chain of thought prompt on top of the simple prompt.
258class FewShotChainOfThoughtPromptBuilder(FewShotPromptBuilder): 259 """A prompt builder that includes a chain of thought prompt on top of the few shot prompt.""" 260 261 def chain_of_thought_prompt(self) -> str | None: 262 return chain_of_thought_prompt(self.task)
A prompt builder that includes a chain of thought prompt on top of the few shot prompt.
265class MultiShotChainOfThoughtPromptBuilder(MultiShotPromptBuilder): 266 """A prompt builder that includes a chain of thought prompt on top of the multi shot prompt.""" 267 268 def chain_of_thought_prompt(self) -> str | None: 269 return chain_of_thought_prompt(self.task)
A prompt builder that includes a chain of thought prompt on top of the multi shot prompt.
272class SavedPromptBuilder(BasePromptBuilder): 273 """A prompt builder that looks up a static prompt.""" 274 275 def __init__(self, task: Task, prompt_id: str): 276 super().__init__(task) 277 prompt_model = next( 278 ( 279 prompt 280 for prompt in task.prompts(readonly=True) 281 if prompt.id == prompt_id 282 ), 283 None, 284 ) 285 if not prompt_model: 286 raise ValueError(f"Prompt ID not found: {prompt_id}") 287 self.prompt_model = prompt_model 288 289 def prompt_id(self) -> str | None: 290 return self.prompt_model.id 291 292 def build_base_prompt(self) -> str: 293 """Returns a saved prompt. 294 295 Returns: 296 str: The prompt string. 297 """ 298 return self.prompt_model.prompt 299 300 def chain_of_thought_prompt(self) -> str | None: 301 return self.prompt_model.chain_of_thought_instructions
A prompt builder that looks up a static prompt.
275 def __init__(self, task: Task, prompt_id: str): 276 super().__init__(task) 277 prompt_model = next( 278 ( 279 prompt 280 for prompt in task.prompts(readonly=True) 281 if prompt.id == prompt_id 282 ), 283 None, 284 ) 285 if not prompt_model: 286 raise ValueError(f"Prompt ID not found: {prompt_id}") 287 self.prompt_model = prompt_model
Initialize the prompt builder with a task.
Args: task (Task): The task containing instructions and requirements.
Returns the ID of the prompt, scoped to this builder.
Returns: str | None: The ID of the prompt, or None if not set.
292 def build_base_prompt(self) -> str: 293 """Returns a saved prompt. 294 295 Returns: 296 str: The prompt string. 297 """ 298 return self.prompt_model.prompt
Returns a saved prompt.
Returns: str: The prompt string.
300 def chain_of_thought_prompt(self) -> str | None: 301 return self.prompt_model.chain_of_thought_instructions
Build and return the chain of thought prompt string.
Returns: str: The constructed chain of thought prompt.
Inherited Members
304class FineTunePromptBuilder(BasePromptBuilder): 305 """A prompt builder that looks up a fine-tune prompt.""" 306 307 def __init__(self, task: Task, nested_fine_tune_id: str): 308 super().__init__(task) 309 310 # IDs are in project_id::task_id::fine_tune_id format 311 self.full_fine_tune_id = nested_fine_tune_id 312 parts = nested_fine_tune_id.split("::") 313 if len(parts) != 3: 314 raise ValueError( 315 f"Invalid fine-tune ID format. Expected 'project_id::task_id::fine_tune_id', got: {nested_fine_tune_id}" 316 ) 317 fine_tune_id = parts[2] 318 319 fine_tune_model = next( 320 ( 321 fine_tune 322 for fine_tune in task.finetunes(readonly=True) 323 if fine_tune.id == fine_tune_id 324 ), 325 None, 326 ) 327 if not fine_tune_model: 328 raise ValueError(f"Fine-tune ID not found: {fine_tune_id}") 329 self.fine_tune_model = fine_tune_model 330 331 def prompt_id(self) -> str | None: 332 return self.full_fine_tune_id 333 334 def build_base_prompt(self) -> str: 335 return self.fine_tune_model.system_message 336 337 def chain_of_thought_prompt(self) -> str | None: 338 return self.fine_tune_model.thinking_instructions
A prompt builder that looks up a fine-tune prompt.
307 def __init__(self, task: Task, nested_fine_tune_id: str): 308 super().__init__(task) 309 310 # IDs are in project_id::task_id::fine_tune_id format 311 self.full_fine_tune_id = nested_fine_tune_id 312 parts = nested_fine_tune_id.split("::") 313 if len(parts) != 3: 314 raise ValueError( 315 f"Invalid fine-tune ID format. Expected 'project_id::task_id::fine_tune_id', got: {nested_fine_tune_id}" 316 ) 317 fine_tune_id = parts[2] 318 319 fine_tune_model = next( 320 ( 321 fine_tune 322 for fine_tune in task.finetunes(readonly=True) 323 if fine_tune.id == fine_tune_id 324 ), 325 None, 326 ) 327 if not fine_tune_model: 328 raise ValueError(f"Fine-tune ID not found: {fine_tune_id}") 329 self.fine_tune_model = fine_tune_model
Initialize the prompt builder with a task.
Args: task (Task): The task containing instructions and requirements.
Returns the ID of the prompt, scoped to this builder.
Returns: str | None: The ID of the prompt, or None if not set.
Build and return the complete prompt string.
Returns: str: The constructed prompt.
337 def chain_of_thought_prompt(self) -> str | None: 338 return self.fine_tune_model.thinking_instructions
Build and return the chain of thought prompt string.
Returns: str: The constructed chain of thought prompt.
Inherited Members
355def prompt_builder_from_ui_name(ui_name: str, task: Task) -> BasePromptBuilder: 356 """Convert a name used in the UI to the corresponding prompt builder class. 357 358 Args: 359 ui_name (str): The UI name for the prompt builder type. 360 361 Returns: 362 type[BasePromptBuilder]: The corresponding prompt builder class. 363 364 Raises: 365 ValueError: If the UI name is not recognized. 366 """ 367 368 # Saved prompts are prefixed with "id::" 369 if ui_name.startswith("id::"): 370 prompt_id = ui_name[4:] 371 return SavedPromptBuilder(task, prompt_id) 372 373 # Fine-tune prompts are prefixed with "fine_tune_prompt::" 374 if ui_name.startswith("fine_tune_prompt::"): 375 fine_tune_id = ui_name[18:] 376 return FineTunePromptBuilder(task, fine_tune_id) 377 378 match ui_name: 379 case "basic": 380 return SimplePromptBuilder(task) 381 case "few_shot": 382 return FewShotPromptBuilder(task) 383 case "many_shot": 384 return MultiShotPromptBuilder(task) 385 case "repairs": 386 return RepairsPromptBuilder(task) 387 case "simple_chain_of_thought": 388 return SimpleChainOfThoughtPromptBuilder(task) 389 case "few_shot_chain_of_thought": 390 return FewShotChainOfThoughtPromptBuilder(task) 391 case "multi_shot_chain_of_thought": 392 return MultiShotChainOfThoughtPromptBuilder(task) 393 case _: 394 raise ValueError(f"Unknown prompt builder: {ui_name}")
Convert a name used in the UI to the corresponding prompt builder class.
Args: ui_name (str): The UI name for the prompt builder type.
Returns: type[BasePromptBuilder]: The corresponding prompt builder class.
Raises: ValueError: If the UI name is not recognized.