from datetime import datetime from openai import OpenAI import streamlit as st from streamlit_server_state import server_state from helper.user_management import lock_llm, unlock_llm def gen_llm_response(query, messages_input=[]): """Create the data required for an LLM call""" messages = messages_input.copy() initial_prompt = messages[-1]["content"] # llm model name llm_model_name = ( st.session_state["llm_info"] .loc[ lambda x: x["name"] == st.session_state["selected_llm"], "model_name", ] .values[0] ) client = OpenAI( api_key=st.session_state["llm_api_key"], base_url=st.session_state["llm_info"] .loc[ lambda x: x["name"] == st.session_state["selected_llm"], "llm_url", ] .values[0], ) # is openrouter reasoning st.session_state["openrouter_reasoning"] = st.session_state[ "is_reasoning_model" ] and "openrouter" in ( st.session_state["llm_info"] .loc[ lambda x: x["name"] == st.session_state["selected_llm"], "llm_url", ] .values[0] ) # modifying prompt to remove time messages[-1]["content"] = messages[-1]["content"].split("
")[0] # adding RAG if necessary context_length = int( st.session_state["llm_info"] .loc[ lambda x: x["name"] == st.session_state["selected_llm"], "context_length", ] .values[0] ) chunk_size = int( st.session_state["settings"] .loc[lambda x: x["field"] == "chunk_size", "value"] .values[0] ) top_n = int( context_length / chunk_size * float( st.session_state["settings"] .loc[lambda x: x["field"] == "context_window_rag_ratio", "value"] .values[0] ) ) if st.session_state["selected_corpus"] != "No corpus": if not ( any( sub in messages[-1]["content"] for sub in ["Given this chat history", "Given this past exchange"] ) ): # don't run for naming the chat # logic for a condensed standalone query if (len(messages) > 2) and ( st.session_state["settings"] .loc[lambda x: x["field"] == "use_condensed_lvs_query", "value"] .values[0] == "1" ): # 2 = 1 system, 1 user prompt condensed_messages = messages.copy() condensed_messages[-1][ "content" ] = f"Given this past exchange, edit this question so that it stands alone in terms of context. Return only the reformulated question, nothing else. Here is the question: '{condensed_messages[-1]['content']}'" condense_response = client.chat.completions.create( model=llm_model_name, temperature=0.0, max_tokens=st.session_state["max_tokens"], messages=condensed_messages.copy(), ) try: condensed_query = condense_response.choices[ 0 ].message.content.split("")[1] except: condensed_query = condense_response.choices[0].message.content else: condensed_query = messages[-1]["content"] text_ids = list( st.session_state["display_metadata"] .loc[lambda x: x["Include in queries"] == True, "text_id"] .values ) lvs_context = server_state["lvs_corpora"][ st.session_state["selected_corpus_realname"] ].get_top_n( condensed_query, top_n=top_n, distance_metric="cosine", text_ids=text_ids, ) st.session_state["latest_chunk_ids"] = lvs_context["chunk_ids"] messages[-1][ "content" ] = f"""{initial_prompt} \n\n {lvs_context["response"]}""" # final llm response if st.session_state["openrouter_reasoning"]: response = client.chat.completions.create( model=llm_model_name, temperature=st.session_state["temperature"], max_tokens=st.session_state["max_tokens"], messages=messages.copy(), reasoning_effort="medium", stream=True, ) else: response = client.chat.completions.create( model=llm_model_name, temperature=st.session_state["temperature"], max_tokens=st.session_state["max_tokens"], messages=messages.copy(), stream=True, ) messages[-1]["content"] = initial_prompt n_fail = 0 # for openrouter reasoning first_token = 0 for chunk in response: first_token += 1 # openrouter reasoning if st.session_state["openrouter_reasoning"]: try: if chunk.choices[0].delta.reasoning is not None: yield chunk.choices[0].delta.reasoning else: if first_token != 1: n_fail += 1 if n_fail == 1: yield "" + chunk.choices[0].delta.content else: if chunk.choices[0].delta.content is not None: yield chunk.choices[0].delta.content except: pass else: if chunk.choices[0].delta.content is not None: yield chunk.choices[0].delta.content yield f"""
{datetime.now().strftime("%Y-%m-%d %H:%M")}""" def write_stream(stream): "write out the stream of the LLM's answer" with st.spinner("Thinking...", show_time=True): if ( ".gguf" in st.session_state["llm_info"] .loc[ lambda x: x["name"] == st.session_state["selected_llm"], "model_name", ] .values[0] ) and ( st.session_state["settings"] .loc[lambda x: x["field"] == "manage_llama_cpp", "value"] .values[0] == "1" ): lock_llm() st.session_state["llm_answer"] = "" st.session_state["reasoning"] = "" # initialize an llm response in case interruption during reasoning st.session_state["chat_history"][st.session_state["selected_chat_id"]][ "reasoning" ] += [st.session_state["reasoning"]] st.session_state["chat_history"][st.session_state["selected_chat_id"]][ "corpus" ] += [st.session_state["selected_corpus"]] st.session_state["chat_history"][st.session_state["selected_chat_id"]][ "chunk_ids" ] += [[]] st.session_state["chat_history"][st.session_state["selected_chat_id"]][ "selected_llm" ] += [st.session_state["selected_llm"]] st.session_state["chat_history"][st.session_state["selected_chat_id"]][ "model_style" ] += [st.session_state["temperature_string"]] st.session_state["chat_history"][st.session_state["selected_chat_id"]][ "messages" ] += [ { "role": "assistant", "content": "", } ] # don't include time in chat history st.session_state["chat_history"][st.session_state["selected_chat_id"]][ "times" ] += [ f"""
{datetime.now().strftime("%Y-%m-%d %H:%M")}""" ] if st.session_state["is_reasoning_model"]: with st.expander("Reasoning", expanded=True): container = st.empty() for chunk in stream: st.session_state["reasoning"] += chunk if "" not in st.session_state["reasoning"]: container.write( f'{st.session_state["reasoning"]}', unsafe_allow_html=True, ) # add to reasoning history st.session_state["chat_history"][ st.session_state["selected_chat_id"] ]["reasoning"][-1] = st.session_state["reasoning"] # add to chunk_ids history if ( len( st.session_state["chat_history"][ st.session_state["selected_chat_id"] ]["chunk_ids"][-1] ) == 0 ) and (st.session_state["selected_corpus"] != "No corpus"): st.session_state["chat_history"][ st.session_state["selected_chat_id"] ]["chunk_ids"][-1] = st.session_state["latest_chunk_ids"] else: if st.session_state[ "openrouter_reasoning" ]: # have to maintain first non-reasoning chunk st.session_state["llm_answer"] += chunk.split("")[1] break # normal LLM output container = st.empty() for chunk in stream: st.session_state["llm_answer"] += chunk container.write(st.session_state["llm_answer"], unsafe_allow_html=True) st.session_state["chat_history"][st.session_state["selected_chat_id"]][ "messages" ][-1] = { "role": "assistant", "content": st.session_state["llm_answer"].split("
")[0], } st.session_state["chat_history"][st.session_state["selected_chat_id"]][ "times" ][ -1 ] = f"""
{datetime.now().strftime("%Y-%m-%d %H:%M")}""" # add to chunk_ids history if ( len( st.session_state["chat_history"][ st.session_state["selected_chat_id"] ]["chunk_ids"][-1] ) == 0 ) and (st.session_state["selected_corpus"] != "No corpus"): st.session_state["chat_history"][st.session_state["selected_chat_id"]][ "chunk_ids" ][-1] = st.session_state["latest_chunk_ids"] if ( ".gguf" in st.session_state["llm_info"] .loc[ lambda x: x["name"] == st.session_state["selected_llm"], "model_name", ] .values[0] ) and ( st.session_state["settings"] .loc[lambda x: x["field"] == "manage_llama_cpp", "value"] .values[0] == "1" ): unlock_llm()