added multiple ollama models
casey
parent
4fd5b9de32
commit
d4cf235347
|
|
@ -2,7 +2,7 @@
|
|||
"cells": [
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 46,
|
||||
"execution_count": 5,
|
||||
"id": "0bf0e410-0031-4983-a868-f2f1253f6947",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
|
|
@ -18,29 +18,39 @@
|
|||
" temperature: float = 0.0\n",
|
||||
" max_tokens: Optional[int] = None\n",
|
||||
" max_retries: int = 3\n",
|
||||
" base_url: str = 'http://192.168.20.30:7869/v1' ## Added as all models being used in this case are served by Ollama\n",
|
||||
"\n",
|
||||
"class LlamaSettings(LLMProviderSettings):\n",
|
||||
"class Llama3Settings(LLMProviderSettings):\n",
|
||||
" api_key: str = \"key\" # required, but not used\n",
|
||||
" # default_model: str = \"phi3:3.8b\"\n",
|
||||
" # default_model: str = \"llama3.2:latest\"\n",
|
||||
" # default_model: str = \"deepseek-coder-v2:latest\"\n",
|
||||
" # default_model: str = \"gemma2:27b\"\n",
|
||||
" # default_model: str = \"solar-pro:latest\" # Need to upgrade Ollama\n",
|
||||
" # default_model: str = \"phi3:14b\"\n",
|
||||
" default_model: str = \"qwen2:0.5b\" # String validation errors \n",
|
||||
" default_model: str = \"llama3.2:latest\"\n",
|
||||
"\n",
|
||||
"class Phi3b8Settings(LLMProviderSettings):\n",
|
||||
" api_key: str = \"key\" # required, but not used\n",
|
||||
" default_model: str = \"phi3:3.8b\" \n",
|
||||
" \n",
|
||||
"class Phi14bSettings(LLMProviderSettings):\n",
|
||||
" api_key: str = \"key\" # required, but not used\n",
|
||||
" default_model: str = \"phi3:14b\"\n",
|
||||
" \n",
|
||||
"class DeepSeekSettings(LLMProviderSettings):\n",
|
||||
" api_key: str = \"key\" # required, but not used\n",
|
||||
" default_model: str = \"deepseek-coder-v2:latest\"\n",
|
||||
" \n",
|
||||
" \n",
|
||||
"class GemmaSettings(LLMProviderSettings):\n",
|
||||
" api_key: str = \"key\" # required, but not used\n",
|
||||
" default_model: str = \"gemma2:27b\"\n",
|
||||
" \n",
|
||||
" \n",
|
||||
" base_url: str = 'http://192.168.20.30:7869/v1'\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"class Settings(BaseSettings):\n",
|
||||
" app_name: str = \"GenAI Project Template\"\n",
|
||||
" # openai: OpenAISettings = OpenAISettings()\n",
|
||||
" # anthropic: AnthropicSettings = AnthropicSettings()\n",
|
||||
" llama: LlamaSettings = LlamaSettings()\n",
|
||||
" \n",
|
||||
" llama: Llama3Settings = Llama3Settings()\n",
|
||||
" phi3b8: Phi3b8Settings = Phi3b8Settings()\n",
|
||||
" phi14b: Phi14bSettings = Phi14bSettings()\n",
|
||||
" deepseek: DeepSeekSettings = DeepSeekSettings()\n",
|
||||
" gemma: GemmaSettings = GemmaSettings()\n",
|
||||
"\n",
|
||||
"@lru_cache\n",
|
||||
"def get_settings():\n",
|
||||
|
|
@ -49,42 +59,10 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 47,
|
||||
"execution_count": 6,
|
||||
"id": "f4993f4b-398e-4ad3-938f-5421677939c4",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"ename": "InstructorRetryException",
|
||||
"evalue": "2 validation errors for CompletionModel\nresponse\n Input should be a valid string [type=string_type, input_value={'reasoning': 'To find ou...s) * (rate per shirt).'}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.10/v/string_type\nreasoning\n Field required [type=missing, input_value={'response': {'reasoning'...odel', 'type': 'object'}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.10/v/missing",
|
||||
"output_type": "error",
|
||||
"traceback": [
|
||||
"\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
|
||||
"\u001b[0;31mValidationError\u001b[0m Traceback (most recent call last)",
|
||||
"File \u001b[0;32m~/miniconda3/envs/LLMatie/lib/python3.13/site-packages/instructor/retry.py:178\u001b[0m, in \u001b[0;36mretry_sync\u001b[0;34m(func, response_model, args, kwargs, context, max_retries, strict, mode, hooks)\u001b[0m\n\u001b[1;32m 172\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m handle_reask_kwargs(\n\u001b[1;32m 173\u001b[0m kwargs\u001b[38;5;241m=\u001b[39mkwargs,\n\u001b[1;32m 174\u001b[0m mode\u001b[38;5;241m=\u001b[39mmode,\n\u001b[1;32m 175\u001b[0m response\u001b[38;5;241m=\u001b[39mresponse,\n\u001b[1;32m 176\u001b[0m exception\u001b[38;5;241m=\u001b[39me,\n\u001b[1;32m 177\u001b[0m )\n\u001b[0;32m--> 178\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m e\n\u001b[1;32m 179\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m RetryError \u001b[38;5;28;01mas\u001b[39;00m e:\n",
|
||||
"File \u001b[0;32m~/miniconda3/envs/LLMatie/lib/python3.13/site-packages/instructor/retry.py:161\u001b[0m, in \u001b[0;36mretry_sync\u001b[0;34m(func, response_model, args, kwargs, context, max_retries, strict, mode, hooks)\u001b[0m\n\u001b[1;32m 158\u001b[0m response \u001b[38;5;241m=\u001b[39m update_total_usage(\n\u001b[1;32m 159\u001b[0m response\u001b[38;5;241m=\u001b[39mresponse, total_usage\u001b[38;5;241m=\u001b[39mtotal_usage\n\u001b[1;32m 160\u001b[0m )\n\u001b[0;32m--> 161\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mprocess_response\u001b[49m\u001b[43m(\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# type: ignore\u001b[39;49;00m\n\u001b[1;32m 162\u001b[0m \u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresponse\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 163\u001b[0m \u001b[43m \u001b[49m\u001b[43mresponse_model\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresponse_model\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 164\u001b[0m \u001b[43m \u001b[49m\u001b[43mvalidation_context\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontext\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 165\u001b[0m \u001b[43m \u001b[49m\u001b[43mstrict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstrict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 166\u001b[0m \u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 167\u001b[0m \u001b[43m \u001b[49m\u001b[43mstream\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mstream\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43;01mFalse\u001b[39;49;00m\u001b[43m)\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 168\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 169\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m (ValidationError, JSONDecodeError) \u001b[38;5;28;01mas\u001b[39;00m e:\n",
|
||||
"File \u001b[0;32m~/miniconda3/envs/LLMatie/lib/python3.13/site-packages/instructor/process_response.py:164\u001b[0m, in \u001b[0;36mprocess_response\u001b[0;34m(response, response_model, stream, validation_context, strict, mode)\u001b[0m\n\u001b[1;32m 162\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m model\n\u001b[0;32m--> 164\u001b[0m model \u001b[38;5;241m=\u001b[39m \u001b[43mresponse_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfrom_response\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 165\u001b[0m \u001b[43m \u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 166\u001b[0m \u001b[43m \u001b[49m\u001b[43mvalidation_context\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvalidation_context\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 167\u001b[0m \u001b[43m \u001b[49m\u001b[43mstrict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstrict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 168\u001b[0m \u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 169\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 171\u001b[0m \u001b[38;5;66;03m# ? This really hints at the fact that we need a better way of\u001b[39;00m\n\u001b[1;32m 172\u001b[0m \u001b[38;5;66;03m# ? attaching usage data and the raw response to the model we return.\u001b[39;00m\n",
|
||||
"File \u001b[0;32m~/miniconda3/envs/LLMatie/lib/python3.13/site-packages/instructor/function_calls.py:166\u001b[0m, in \u001b[0;36mOpenAISchema.from_response\u001b[0;34m(cls, completion, validation_context, strict, mode)\u001b[0m\n\u001b[1;32m 158\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m mode \u001b[38;5;129;01min\u001b[39;00m {\n\u001b[1;32m 159\u001b[0m Mode\u001b[38;5;241m.\u001b[39mJSON,\n\u001b[1;32m 160\u001b[0m Mode\u001b[38;5;241m.\u001b[39mJSON_SCHEMA,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 164\u001b[0m Mode\u001b[38;5;241m.\u001b[39mFIREWORKS_JSON,\n\u001b[1;32m 165\u001b[0m }:\n\u001b[0;32m--> 166\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparse_json\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcompletion\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mvalidation_context\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstrict\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 168\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mInvalid patch mode: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mmode\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
|
||||
"File \u001b[0;32m~/miniconda3/envs/LLMatie/lib/python3.13/site-packages/instructor/function_calls.py:384\u001b[0m, in \u001b[0;36mOpenAISchema.parse_json\u001b[0;34m(cls, completion, validation_context, strict)\u001b[0m\n\u001b[1;32m 382\u001b[0m message \u001b[38;5;241m=\u001b[39m extract_json_from_codeblock(message)\n\u001b[0;32m--> 384\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mmodel_validate_json\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 385\u001b[0m \u001b[43m \u001b[49m\u001b[43mmessage\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 386\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvalidation_context\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 387\u001b[0m \u001b[43m \u001b[49m\u001b[43mstrict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstrict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 388\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[0;32m~/miniconda3/envs/LLMatie/lib/python3.13/site-packages/pydantic/main.py:656\u001b[0m, in \u001b[0;36mBaseModel.model_validate_json\u001b[0;34m(cls, json_data, strict, context)\u001b[0m\n\u001b[1;32m 655\u001b[0m __tracebackhide__ \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mTrue\u001b[39;00m\n\u001b[0;32m--> 656\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mcls\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m__pydantic_validator__\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mvalidate_json\u001b[49m\u001b[43m(\u001b[49m\u001b[43mjson_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mstrict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstrict\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontext\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"\u001b[0;31mValidationError\u001b[0m: 2 validation errors for CompletionModel\nresponse\n Input should be a valid string [type=string_type, input_value={'reasoning': 'To find ou...s) * (rate per shirt).'}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.10/v/string_type\nreasoning\n Field required [type=missing, input_value={'response': {'reasoning'...odel', 'type': 'object'}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.10/v/missing",
|
||||
"\nThe above exception was the direct cause of the following exception:\n",
|
||||
"\u001b[0;31mRetryError\u001b[0m Traceback (most recent call last)",
|
||||
"File \u001b[0;32m~/miniconda3/envs/LLMatie/lib/python3.13/site-packages/instructor/retry.py:151\u001b[0m, in \u001b[0;36mretry_sync\u001b[0;34m(func, response_model, args, kwargs, context, max_retries, strict, mode, hooks)\u001b[0m\n\u001b[1;32m 150\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[0;32m--> 151\u001b[0m \u001b[43m\u001b[49m\u001b[38;5;28;43;01mfor\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mattempt\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;129;43;01min\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[43m:\u001b[49m\n\u001b[1;32m 152\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;28;43;01mwith\u001b[39;49;00m\u001b[43m \u001b[49m\u001b[43mattempt\u001b[49m\u001b[43m:\u001b[49m\n",
|
||||
"File \u001b[0;32m~/miniconda3/envs/LLMatie/lib/python3.13/site-packages/tenacity/__init__.py:443\u001b[0m, in \u001b[0;36mBaseRetrying.__iter__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m 442\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28;01mTrue\u001b[39;00m:\n\u001b[0;32m--> 443\u001b[0m do \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43miter\u001b[49m\u001b[43m(\u001b[49m\u001b[43mretry_state\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mretry_state\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 444\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(do, DoAttempt):\n",
|
||||
"File \u001b[0;32m~/miniconda3/envs/LLMatie/lib/python3.13/site-packages/tenacity/__init__.py:376\u001b[0m, in \u001b[0;36mBaseRetrying.iter\u001b[0;34m(self, retry_state)\u001b[0m\n\u001b[1;32m 375\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m action \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39miter_state\u001b[38;5;241m.\u001b[39mactions:\n\u001b[0;32m--> 376\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[43maction\u001b[49m\u001b[43m(\u001b[49m\u001b[43mretry_state\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 377\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m result\n",
|
||||
"File \u001b[0;32m~/miniconda3/envs/LLMatie/lib/python3.13/site-packages/tenacity/__init__.py:419\u001b[0m, in \u001b[0;36mBaseRetrying._post_stop_check_actions.<locals>.exc_check\u001b[0;34m(rs)\u001b[0m\n\u001b[1;32m 418\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m retry_exc\u001b[38;5;241m.\u001b[39mreraise()\n\u001b[0;32m--> 419\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m retry_exc \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mfut\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mexception\u001b[39;00m()\n",
|
||||
"\u001b[0;31mRetryError\u001b[0m: RetryError[<Future at 0x7a6435069450 state=finished raised ValidationError>]",
|
||||
"\nThe above exception was the direct cause of the following exception:\n",
|
||||
"\u001b[0;31mInstructorRetryException\u001b[0m Traceback (most recent call last)",
|
||||
"Cell \u001b[0;32mIn[47], line 57\u001b[0m\n\u001b[1;32m 48\u001b[0m messages \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m 49\u001b[0m {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrole\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msystem\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcontent\u001b[39m\u001b[38;5;124m\"\u001b[39m: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mYou are a helpful assistant.\u001b[39m\u001b[38;5;124m\"\u001b[39m},\n\u001b[1;32m 50\u001b[0m {\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 53\u001b[0m },\n\u001b[1;32m 54\u001b[0m ]\n\u001b[1;32m 56\u001b[0m llm \u001b[38;5;241m=\u001b[39m LLMFactory(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mllama\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m---> 57\u001b[0m completion \u001b[38;5;241m=\u001b[39m \u001b[43mllm\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_completion\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 58\u001b[0m \u001b[43m \u001b[49m\u001b[43mresponse_model\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mCompletionModel\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 59\u001b[0m \u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmessages\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 60\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 61\u001b[0m \u001b[38;5;28;01massert\u001b[39;00m \u001b[38;5;28misinstance\u001b[39m(completion, CompletionModel)\n\u001b[1;32m 63\u001b[0m \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mResponse: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mcompletion\u001b[38;5;241m.\u001b[39mresponse\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
|
||||
"Cell \u001b[0;32mIn[47], line 39\u001b[0m, in \u001b[0;36mLLMFactory.create_completion\u001b[0;34m(self, response_model, messages, **kwargs)\u001b[0m\n\u001b[1;32m 28\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcreate_completion\u001b[39m(\n\u001b[1;32m 29\u001b[0m \u001b[38;5;28mself\u001b[39m, response_model: Type[BaseModel], messages: List[Dict[\u001b[38;5;28mstr\u001b[39m, \u001b[38;5;28mstr\u001b[39m]], \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m 30\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Any:\n\u001b[1;32m 31\u001b[0m completion_params \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m 32\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel\u001b[39m\u001b[38;5;124m\"\u001b[39m: kwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmodel\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msettings\u001b[38;5;241m.\u001b[39mdefault_model),\n\u001b[1;32m 33\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtemperature\u001b[39m\u001b[38;5;124m\"\u001b[39m: kwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtemperature\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39msettings\u001b[38;5;241m.\u001b[39mtemperature),\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 37\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmessages\u001b[39m\u001b[38;5;124m\"\u001b[39m: messages,\n\u001b[1;32m 38\u001b[0m }\n\u001b[0;32m---> 39\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mclient\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mchat\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcompletions\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mcompletion_params\u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[0;32m~/miniconda3/envs/LLMatie/lib/python3.13/site-packages/instructor/client.py:176\u001b[0m, in \u001b[0;36mInstructor.create\u001b[0;34m(self, response_model, messages, max_retries, validation_context, context, strict, **kwargs)\u001b[0m\n\u001b[1;32m 164\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mcreate\u001b[39m(\n\u001b[1;32m 165\u001b[0m \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m 166\u001b[0m response_model: \u001b[38;5;28mtype\u001b[39m[T] \u001b[38;5;241m|\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m,\n\u001b[0;32m (...)\u001b[0m\n\u001b[1;32m 172\u001b[0m \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs: Any,\n\u001b[1;32m 173\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m T \u001b[38;5;241m|\u001b[39m Any \u001b[38;5;241m|\u001b[39m Awaitable[T] \u001b[38;5;241m|\u001b[39m Awaitable[Any]:\n\u001b[1;32m 174\u001b[0m kwargs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mhandle_kwargs(kwargs)\n\u001b[0;32m--> 176\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcreate_fn\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 177\u001b[0m \u001b[43m \u001b[49m\u001b[43mresponse_model\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresponse_model\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 178\u001b[0m \u001b[43m \u001b[49m\u001b[43mmessages\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmessages\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 179\u001b[0m \u001b[43m \u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 180\u001b[0m \u001b[43m \u001b[49m\u001b[43mvalidation_context\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mvalidation_context\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 181\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontext\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 182\u001b[0m \u001b[43m \u001b[49m\u001b[43mstrict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstrict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 183\u001b[0m \u001b[43m \u001b[49m\u001b[43mhooks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mhooks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 184\u001b[0m \u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 185\u001b[0m \u001b[43m \u001b[49m\u001b[43m)\u001b[49m\n",
|
||||
"File \u001b[0;32m~/miniconda3/envs/LLMatie/lib/python3.13/site-packages/instructor/patch.py:193\u001b[0m, in \u001b[0;36mpatch.<locals>.new_create_sync\u001b[0;34m(response_model, validation_context, context, max_retries, strict, hooks, *args, **kwargs)\u001b[0m\n\u001b[1;32m 187\u001b[0m response_model, new_kwargs \u001b[38;5;241m=\u001b[39m handle_response_model(\n\u001b[1;32m 188\u001b[0m response_model\u001b[38;5;241m=\u001b[39mresponse_model, mode\u001b[38;5;241m=\u001b[39mmode, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs\n\u001b[1;32m 189\u001b[0m ) \u001b[38;5;66;03m# type: ignore\u001b[39;00m\n\u001b[1;32m 191\u001b[0m new_kwargs \u001b[38;5;241m=\u001b[39m handle_templating(new_kwargs, context)\n\u001b[0;32m--> 193\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[43mretry_sync\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m 194\u001b[0m \u001b[43m \u001b[49m\u001b[43mfunc\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mfunc\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;66;43;03m# type: ignore\u001b[39;49;00m\n\u001b[1;32m 195\u001b[0m \u001b[43m \u001b[49m\u001b[43mresponse_model\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mresponse_model\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 196\u001b[0m \u001b[43m \u001b[49m\u001b[43mcontext\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcontext\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 197\u001b[0m \u001b[43m \u001b[49m\u001b[43mmax_retries\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmax_retries\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 198\u001b[0m \u001b[43m \u001b[49m\u001b[43margs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 199\u001b[0m \u001b[43m \u001b[49m\u001b[43mhooks\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhooks\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 200\u001b[0m \u001b[43m \u001b[49m\u001b[43mstrict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstrict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 201\u001b[0m \u001b[43m \u001b[49m\u001b[43mkwargs\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnew_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 202\u001b[0m \u001b[43m \u001b[49m\u001b[43mmode\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmode\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m 203\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m 204\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
|
||||
"File \u001b[0;32m~/miniconda3/envs/LLMatie/lib/python3.13/site-packages/instructor/retry.py:181\u001b[0m, in \u001b[0;36mretry_sync\u001b[0;34m(func, response_model, args, kwargs, context, max_retries, strict, mode, hooks)\u001b[0m\n\u001b[1;32m 179\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m RetryError \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m 180\u001b[0m logger\u001b[38;5;241m.\u001b[39mdebug(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mRetry error: \u001b[39m\u001b[38;5;132;01m{\u001b[39;00me\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[0;32m--> 181\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m InstructorRetryException(\n\u001b[1;32m 182\u001b[0m e\u001b[38;5;241m.\u001b[39mlast_attempt\u001b[38;5;241m.\u001b[39m_exception,\n\u001b[1;32m 183\u001b[0m last_completion\u001b[38;5;241m=\u001b[39mresponse,\n\u001b[1;32m 184\u001b[0m n_attempts\u001b[38;5;241m=\u001b[39mattempt\u001b[38;5;241m.\u001b[39mretry_state\u001b[38;5;241m.\u001b[39mattempt_number,\n\u001b[1;32m 185\u001b[0m \u001b[38;5;66;03m#! deprecate messages soon\u001b[39;00m\n\u001b[1;32m 186\u001b[0m messages\u001b[38;5;241m=\u001b[39mkwargs\u001b[38;5;241m.\u001b[39mget(\n\u001b[1;32m 187\u001b[0m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mmessages\u001b[39m\u001b[38;5;124m\"\u001b[39m, kwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcontents\u001b[39m\u001b[38;5;124m\"\u001b[39m, kwargs\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mchat_history\u001b[39m\u001b[38;5;124m\"\u001b[39m, []))\n\u001b[1;32m 188\u001b[0m ),\n\u001b[1;32m 189\u001b[0m create_kwargs\u001b[38;5;241m=\u001b[39mkwargs,\n\u001b[1;32m 190\u001b[0m total_usage\u001b[38;5;241m=\u001b[39mtotal_usage,\n\u001b[1;32m 191\u001b[0m ) \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01me\u001b[39;00m\n",
|
||||
"\u001b[0;31mInstructorRetryException\u001b[0m: 2 validation errors for CompletionModel\nresponse\n Input should be a valid string [type=string_type, input_value={'reasoning': 'To find ou...s) * (rate per shirt).'}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.10/v/string_type\nreasoning\n Field required [type=missing, input_value={'response': {'reasoning'...odel', 'type': 'object'}, input_type=dict]\n For further information visit https://errors.pydantic.dev/2.10/v/missing"
|
||||
]
|
||||
}
|
||||
],
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"from typing import Any, Dict, List, Type\n",
|
||||
"\n",
|
||||
|
|
@ -100,12 +78,29 @@
|
|||
" self.settings = getattr(get_settings(), provider)\n",
|
||||
" self.client = self._initialize_client()\n",
|
||||
"\n",
|
||||
" \n",
|
||||
" def _initialize_client(self) -> Any:\n",
|
||||
" client_initializers = {\n",
|
||||
" \"llama\": lambda s: instructor.from_openai(\n",
|
||||
" OpenAI(base_url=s.base_url, api_key=s.api_key),\n",
|
||||
" mode=instructor.Mode.JSON,\n",
|
||||
" ),\n",
|
||||
" \"phi3b8\": lambda s: instructor.from_openai(\n",
|
||||
" OpenAI(base_url=s.base_url, api_key=s.api_key),\n",
|
||||
" mode=instructor.Mode.JSON,\n",
|
||||
" ),\n",
|
||||
" \"phi14b\": lambda s: instructor.from_openai(\n",
|
||||
" OpenAI(base_url=s.base_url, api_key=s.api_key),\n",
|
||||
" mode=instructor.Mode.JSON,\n",
|
||||
" ),\n",
|
||||
" \"deepseek\": lambda s: instructor.from_openai(\n",
|
||||
" OpenAI(base_url=s.base_url, api_key=s.api_key),\n",
|
||||
" mode=instructor.Mode.JSON,\n",
|
||||
" ),\n",
|
||||
" \"gemma\": lambda s: instructor.from_openai(\n",
|
||||
" OpenAI(base_url=s.base_url, api_key=s.api_key),\n",
|
||||
" mode=instructor.Mode.JSON,\n",
|
||||
" ),\n",
|
||||
" }\n",
|
||||
"\n",
|
||||
" initializer = client_initializers.get(self.provider)\n",
|
||||
|
|
@ -124,38 +119,53 @@
|
|||
" \"response_model\": response_model,\n",
|
||||
" \"messages\": messages,\n",
|
||||
" }\n",
|
||||
" return self.client.chat.completions.create(**completion_params)\n",
|
||||
" return self.client.chat.completions.create(**completion_params)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"id": "fa7ea636-7abe-4dc4-9665-1eb9bc4fde17",
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Response: It will still take approximately 2 hours to dry 5 shirts.\n",
|
||||
"\n",
|
||||
"Reasoning: Drying time for clothes in the sun is primarily determined by factors like sunlight intensity, temperature, and humidity. Assuming these conditions remain constant, adding more shirts won't significantly increase the drying time. Each shirt will dry independently of the others.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"class CompletionModel(BaseModel):\n",
|
||||
" response: str = Field(description=\"Your response to the user.\")\n",
|
||||
" reasoning: str = Field(description=\"Explain your reasoning for the response.\")\n",
|
||||
"\n",
|
||||
"messages = [\n",
|
||||
" {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
|
||||
" {\n",
|
||||
" \"role\": \"user\",\n",
|
||||
" \"content\": \"If it takes 2 hours to dry 1 shirt out in the sun, how long will it take to dry 5 shirts?\",\n",
|
||||
" },\n",
|
||||
"]\n",
|
||||
"\n",
|
||||
"if __name__ == \"__main__\":\n",
|
||||
"llm = LLMFactory(\"gemma\")\n",
|
||||
"completion = llm.create_completion(\n",
|
||||
" response_model=CompletionModel,\n",
|
||||
" messages=messages,\n",
|
||||
")\n",
|
||||
"assert isinstance(completion, CompletionModel)\n",
|
||||
"\n",
|
||||
" class CompletionModel(BaseModel):\n",
|
||||
" response: str = Field(description=\"Your response to the user.\")\n",
|
||||
" reasoning: str = Field(description=\"Explain your reasoning for the response.\")\n",
|
||||
"\n",
|
||||
" messages = [\n",
|
||||
" {\"role\": \"system\", \"content\": \"You are a helpful assistant.\"},\n",
|
||||
" {\n",
|
||||
" \"role\": \"user\",\n",
|
||||
" \"content\": \"If it takes 2 hours to dry 1 shirt out in the sun, how long will it take to dry 5 shirts?\",\n",
|
||||
" },\n",
|
||||
" ]\n",
|
||||
"\n",
|
||||
" llm = LLMFactory(\"llama\")\n",
|
||||
" completion = llm.create_completion(\n",
|
||||
" response_model=CompletionModel,\n",
|
||||
" messages=messages,\n",
|
||||
" )\n",
|
||||
" assert isinstance(completion, CompletionModel)\n",
|
||||
"\n",
|
||||
" print(f\"Response: {completion.response}\\n\")\n",
|
||||
" print(f\"Reasoning: {completion.reasoning}\")"
|
||||
"print(f\"Response: {completion.response}\\n\")\n",
|
||||
"print(f\"Reasoning: {completion.reasoning}\")"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": null,
|
||||
"id": "fa7ea636-7abe-4dc4-9665-1eb9bc4fde17",
|
||||
"id": "c42495cd-5e8e-48bc-aaea-e36f18b02baf",
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": []
|
||||
|
|
|
|||
Loading…
Reference in New Issue