Forked from fsndzomga/Pure DSPy Synthetic Prompt Optimization.py
Created
November 17, 2024 20:42
-
-
Save OctAg0nO/9d1c4110a73e345d4c7b70f193d875f3 to your computer and use it in GitHub Desktop.
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Synthetic data generation using DSPy | |
| from typing import List | |
| import dspy | |
| llm = dspy.OpenAI(model='gpt-3.5-turbo',api_key=openai_key) | |
| dspy.settings.configure(lm=llm) | |
| class FactGeneration(dspy.Signature): | |
| """Generate facts and their veracity, it should be different than old data""" | |
| sindex = dspy.InputField(desc="a random string") | |
| fact = dspy.OutputField(desc="a statement") | |
| veracity = dspy.OutputField(desc="a boolean True or False") | |
| fact_generator = dspy.Predict(FactGeneration, n=15) | |
| response = fact_generator(sindex="1") | |
| few_shot_examples: List[dspy.Example] = [ | |
| dspy.Example({'fact': fact, 'answer': veracity}) | |
| for fact, veracity in zip(response.completions.fact, response.completions.veracity) | |
| ] | |
| # Synthetic Prompt Optimization | |
| from dspy.teleprompt import BootstrapFewShot | |
| from dspy.evaluate import answer_exact_match | |
| text = "Barack Obama was not President of the USA" | |
| # define the fact as input to the lie detector | |
| trainset = [x.with_inputs('fact') for x in few_shot_examples] | |
| # define the signature to be used in by the lie detector module | |
| # for the evaluation, you need to define an answer field | |
| class Veracity(dspy.Signature): | |
| "Evaluate the veracity of a statement" | |
| fact = dspy.InputField(desc="a statement") | |
| answer = dspy.OutputField(desc="an assessment of the veracity of the statement") | |
| class lie_detector(dspy.Module): | |
| def __init__(self): | |
| super().__init__() | |
| self.lie_identification = dspy.ChainOfThought(Veracity) | |
| def forward(self, fact): | |
| return self.lie_identification(fact=fact) | |
| teleprompter = BootstrapFewShot(metric=answer_exact_match) | |
| compiled_lie_detector = teleprompter.compile(lie_detector(), trainset=trainset) | |
| response = compiled_lie_detector(fact=text) | |
| print(f"The statement '{text}' is {response.answer}.") |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment