Spaces:
Runtime error
Runtime error
| diff --git a/vllm/entrypoints/openai/serving_chat.py b/vllm/entrypoints/openai/serving_chat.py | |
| index 79eac184a..868e236e6 100644 | |
| --- a/vllm/entrypoints/openai/serving_chat.py | |
| +++ b/vllm/entrypoints/openai/serving_chat.py | |
| @@ -200,6 +200,15 @@ class OpenAIServingChat(OpenAIServing): | |
| logger.exception("Error in preprocessing prompt inputs") | |
| return self.create_error_response(f"{e} {e.__cause__}") | |
| + logger.info("DEBUG "*100) | |
| + | |
| + logger.info("REQUEST PROMPTS %s", request_prompts) | |
| + logger.info("CONVERSATION %s", conversation) | |
| + | |
| + # Not sure to understand what the engine prompt is: from what I understand this is the raw /tokenized input | |
| + # fed to the llm (so what we are looking for) | |
| + logger.info("ENGINE PROMPTS %s", engine_prompts) | |
| + | |
| request_id = "chatcmpl-" \ | |
| f"{self._base_request_id(raw_request, request.request_id)}" | |
| diff --git a/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py | |
| index c7030d34d..7a2765838 100644 | |
| --- a/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py | |
| +++ b/vllm/entrypoints/openai/tool_parsers/hermes_tool_parser.py | |
| @@ -44,7 +44,7 @@ class Hermes2ProToolParser(ToolParser): | |
| self.tool_call_end_token: str = "</tool_call>" | |
| self.tool_call_regex = re.compile( | |
| - r"<tool_call>(.*?)</tool_call>|<tool_call>(.*)", re.DOTALL) | |
| + r"<tool_call>\s*(.*?)\s*(?:<tool_call>\s*|</tool_call>\s*|$)", re.DOTALL) | |
| self.scratch_pad_regex = re.compile( | |
| r"<scratch_pad>(.*?)</scratch_pad>", re.DOTALL) | |
| @@ -80,15 +80,17 @@ class Hermes2ProToolParser(ToolParser): | |
| # tag and end-of-string so the result of | |
| # findall is an array of tuples where one is a function call and | |
| # the other is None | |
| - function_call_tuples = ( | |
| - self.tool_call_regex.findall(model_output)) | |
| - | |
| - # load the JSON, and then use it to build the Function and | |
| - # Tool Call | |
| - raw_function_calls = [ | |
| - json.loads(match[0] if match[0] else match[1]) | |
| - for match in function_call_tuples | |
| - ] | |
| + matches = self.tool_call_regex.findall(model_output) | |
| + raw_function_calls = [] | |
| + for match in matches: | |
| + if not match: | |
| + continue | |
| + try: | |
| + parsed = json.loads(match.strip()) | |
| + raw_function_calls.append(parsed) | |
| + except json.JSONDecodeError as e: | |
| + logger.warning("Skipping malformed tool_call block: %s", e) | |
| + | |
| tool_calls = [ | |
| ToolCall( | |
| type="function", | |
| @@ -99,9 +101,8 @@ class Hermes2ProToolParser(ToolParser): | |
| ensure_ascii=False))) | |
| for function_call in raw_function_calls | |
| ] | |
| - | |
| - content = model_output[:model_output. | |
| - find(self.tool_call_start_token)] | |
| + tool_call_start = model_output.find(self.tool_call_start_token) | |
| + content = model_output[:tool_call_start] if tool_call_start >= 0 else None | |
| return ExtractedToolCallInformation( | |
| tools_called=True, | |
| tool_calls=tool_calls, | |