Skip to content

Commit b333511

Browse files
dhopp1dhopp1
committed
Fixing memory_limit issue and moving context window to llm-level
1 parent 3268215 commit b333511

4 files changed

Lines changed: 97 additions & 38 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,10 @@
11
# Change Log
22

3+
### 0.0.5
4+
### Fixed
5+
* Moved context window to only be handled on an LLM-level.
6+
* Fixed chat memory limit overflowing the context window leading the model to stop answering.
7+
38
### 0.0.4
49
### Added
510
* Added Google News support for generation of a corpus.

‎helper/modelling.py‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,15 @@ def initialize_llm():
7878
]
7979
.values[0],
8080
n_gpu_layers=100,
81+
context_window=st.session_state["llm_dict"]
82+
.loc[
83+
lambda x: x.name
84+
== server_state[
85+
f'{st.session_state["user_name"]}_selected_llm'
86+
],
87+
"context_window",
88+
]
89+
.values[0],
8190
),
8291
)
8392
except:
@@ -168,6 +177,35 @@ def load_rag_pipeline():
168177
):
169178
clear_models()
170179

180+
# update the memory limit
181+
update_server_state(
182+
f'{st.session_state["user_name"]}_memory_limit',
183+
int(
184+
(
185+
1
186+
- server_state[f'{st.session_state["user_name"]}_similarity_top_k']
187+
* server_state[f'{st.session_state["user_name"]}_chunk_size']
188+
/ st.session_state["llm_dict"]
189+
.loc[
190+
lambda x: x.name
191+
== server_state[
192+
f'{st.session_state["user_name"]}_selected_llm'
193+
],
194+
"context_window",
195+
]
196+
.values[0]
197+
)
198+
* st.session_state["llm_dict"]
199+
.loc[
200+
lambda x: x.name
201+
== server_state[f'{st.session_state["user_name"]}_selected_llm'],
202+
"context_window",
203+
]
204+
.values[0]
205+
- 200
206+
),
207+
)
208+
171209
# hid messages so you can see the initializer
172210
if "message_box" in st.session_state:
173211
st.session_state["message_box"].empty()

‎helper/ui.py‎

Lines changed: 52 additions & 32 deletions
Original file line numberDiff line numberDiff line change
@@ -303,7 +303,8 @@ def ui_advanced_model_params():
303303
if f'{st.session_state["user_name"]}_similarity_top_k'
304304
not in server_state
305305
else server_state[f'{st.session_state["user_name"]}_similarity_top_k'],
306-
help="The number of contextual document chunks to retrieve for RAG.",
306+
# help="The number of contextual document chunks to retrieve for RAG.",
307+
help=f"""The number of contextual document chunks to retrieve for RAG. `Similarity top K` * `Chunk size` must be less than your chosen LLM's context window, which is `{st.session_state["llm_dict"].loc[lambda x: x.name == server_state[f'{st.session_state["user_name"]}_selected_llm'], "context_window"].values[0]}`.""",
307308
)
308309

309310
# n_gpu layers
@@ -339,32 +340,6 @@ def ui_advanced_model_params():
339340
help="How long to limit the responses to (token ≈ word).",
340341
)
341342

342-
# context window
343-
with no_rerun:
344-
server_state[f'{st.session_state["user_name"]}_context_window'] = st.slider(
345-
"Context window",
346-
min_value=500,
347-
max_value=50000,
348-
step=100,
349-
value=4000
350-
if f'{st.session_state["user_name"]}_context_window' not in server_state
351-
else server_state[f'{st.session_state["user_name"]}_context_window'],
352-
help="How large to make the context window for the LLM. The maximum depends on the model, a higher value might result in context window too large errors.",
353-
)
354-
355-
# memory limit
356-
with no_rerun:
357-
server_state[f'{st.session_state["user_name"]}_memory_limit'] = st.slider(
358-
"Memory limit",
359-
min_value=80,
360-
max_value=80000,
361-
step=8,
362-
value=2048
363-
if f'{st.session_state["user_name"]}_memory_limit' not in server_state
364-
else server_state[f'{st.session_state["user_name"]}_memory_limit'],
365-
help="How many tokens (words) memory to give the chatbot.",
366-
)
367-
368343
# system prompt
369344
with no_rerun:
370345
server_state[
@@ -419,6 +394,55 @@ def ui_advanced_model_params():
419394
help="Click if you change the `Which LLM` or `Which corpus` options.",
420395
)
421396

397+
# show error if chunk size * top k too large
398+
chunk_error = st.sidebar.empty()
399+
if (
400+
server_state[f'{st.session_state["user_name"]}_chunk_size']
401+
* server_state[f'{st.session_state["user_name"]}_similarity_top_k']
402+
> st.session_state["llm_dict"]
403+
.loc[
404+
lambda x: x.name
405+
== server_state[f'{st.session_state["user_name"]}_selected_llm'],
406+
"context_window",
407+
]
408+
.values[0]
409+
):
410+
chunk_error.error(
411+
f"""Chunk size ({server_state[f'{st.session_state["user_name"]}_chunk_size']}) * similarity top K ({server_state[f'{st.session_state["user_name"]}_similarity_top_k']}) = {server_state[f'{st.session_state["user_name"]}_chunk_size'] * server_state[f'{st.session_state["user_name"]}_similarity_top_k']} is larger than the context window ({st.session_state["llm_dict"].loc[lambda x: x.name == server_state[f'{st.session_state["user_name"]}_selected_llm'], "context_window"].values[0]}). Either decrease chunk size or lower similarity top K."""
412+
)
413+
else:
414+
chunk_error.empty()
415+
416+
# set memory limit dynamically
417+
if f'{st.session_state["user_name"]}_memory_limit' not in server_state:
418+
update_server_state(
419+
f'{st.session_state["user_name"]}_memory_limit',
420+
int(
421+
(
422+
1
423+
- server_state[f'{st.session_state["user_name"]}_similarity_top_k']
424+
* server_state[f'{st.session_state["user_name"]}_chunk_size']
425+
/ st.session_state["llm_dict"]
426+
.loc[
427+
lambda x: x.name
428+
== server_state[
429+
f'{st.session_state["user_name"]}_selected_llm'
430+
],
431+
"context_window",
432+
]
433+
.values[0]
434+
)
435+
* st.session_state["llm_dict"]
436+
.loc[
437+
lambda x: x.name
438+
== server_state[f'{st.session_state["user_name"]}_selected_llm'],
439+
"context_window",
440+
]
441+
.values[0]
442+
- 200
443+
),
444+
)
445+
422446

423447
def ui_reset():
424448
"UI reset button"
@@ -450,7 +474,6 @@ def ui_export_chat_end_session():
450474
del server_state[f'{st.session_state["user_name"]}_similarity_top_k']
451475
del server_state[f'{st.session_state["user_name"]}_temperature']
452476
del server_state[f'{st.session_state["user_name"]}_max_new_tokens']
453-
del server_state[f'{st.session_state["user_name"]}_context_window']
454477
del server_state[f'{st.session_state["user_name"]}_memory_limit']
455478
del server_state[f'{st.session_state["user_name"]}_system_prompt']
456479
del server_state[f'{st.session_state["user_name"]}_chunk_overlap']
@@ -619,9 +642,6 @@ def import_chat():
619642
max_new_tokens=server_state[
620643
f'{st.session_state["user_name"]}_max_new_tokens'
621644
],
622-
context_window=server_state[
623-
f'{st.session_state["user_name"]}_context_window'
624-
],
625645
use_chat_engine=st.session_state["use_chat_engine"],
626646
reset_chat_engine=st.session_state["reset_chat_engine"],
627647
memory_limit=server_state[
@@ -677,7 +697,7 @@ def import_chat():
677697
Similarity top K: {server_state[f'{st.session_state["user_name"]}_similarity_top_k']}
678698
Temperature: {server_state[f'{st.session_state["user_name"]}_temperature']}
679699
Max new tokens: {server_state[f'{st.session_state["user_name"]}_max_new_tokens']}
680-
Context window: {server_state[f'{st.session_state["user_name"]}_context_window']}
700+
Context window: {st.session_state["llm_dict"].loc[lambda x: x.name == server_state[f'{st.session_state["user_name"]}_selected_llm'], "context_window"].values[0]}
681701
Memory limit: {server_state[f'{st.session_state["user_name"]}_memory_limit']}
682702
System prompt: {server_state[f'{st.session_state["user_name"]}_system_prompt']}
683703
Chunk overlap: {server_state[f'{st.session_state["user_name"]}_chunk_overlap']}

‎metadata/llm_list.csv‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,2 @@
1-
name,llm_url,llm_path
2-
llama-2-7b,https://huggingface.co/TheBloke/Llama-2-7B-Chat-GGUF/resolve/main/llama-2-7b-chat.Q5_K_M.gguf,models/llama-2-7b-chat.Q5_K_M.gguf
3-
llama-2-13b,https://huggingface.co/TheBloke/Llama-2-13B-chat-GGUF/resolve/main/llama-2-13b-chat.Q5_K_M.gguf,models/llama-2-13b-chat.Q5_K_M.gguf
4-
mistral-7b,https://huggingface.co/TheBloke/Mistral-7B-Claude-Chat-GGUF/resolve/main/mistral-7b-claude-chat.Q5_K_M.gguf,models/mistral-7b-claude-chat.Q5_K_M.gguf
5-
mistral-docsgpt,https://huggingface.co/TheBloke/docsgpt-7B-mistral-GGUF/resolve/main/docsgpt-7b-mistral.Q5_K_M.gguf,models/docsgpt-7b-mistral.Q5_K_M.gguf
6-
deepseek-coder-7b,https://huggingface.co/TheBloke/deepseek-coder-6.7B-instruct-GGUF/resolve/main/deepseek-coder-6.7b-instruct.Q5_K_M.gguf,models/deepseek-coder-6.7b-instruct.Q5_K_M.gguf
1+
name,llm_url,llm_path,context_window
2+
mistral-docsgpt,https://huggingface.co/TheBloke/docsgpt-7B-mistral-GGUF/resolve/main/docsgpt-7b-mistral.Q6_K.gguf,models/docsgpt-7b-mistral.Q6_K.gguf,4000

0 commit comments

Comments
 (0)