Support leave-output-tensor option

app.py

@@ -196,6 +196,7 @@ def quantize_model(
     imatrix_path: str,
     quant_embedding: bool,
     embedding_tensor_method: str,
+    leave_output: bool,
     quant_output: bool,
     output_tensor_method: str,
 ):
@@ -220,10 +221,12 @@ def quantize_model(
     if quant_embedding:
         quantize_cmd.append("--token-embedding-type")
         quantize_cmd.append(embedding_tensor_method)
-    if quant_output:
-        quantize_cmd.append("--output-tensor-type")
-        quantize_cmd.append(output_tensor_method)
-
+    if leave_output:
+        quantize_cmd.append("--leave-output-tensor")
+    else:
+        if quant_output:
+            quantize_cmd.append("--output-tensor-type")
+            quantize_cmd.append(output_tensor_method)
     if use_imatrix:
         quantize_cmd.append("--imatrix")
         quantize_cmd.append(imatrix_path)
@@ -329,6 +332,7 @@ def process_model(
     gguf_name: str,
     quant_embedding: bool,
     embedding_tensor_method: str,
+    leave_output: bool,
     quant_output: bool,
     output_tensor_method: str,
     split_model: bool,
@@ -349,7 +353,7 @@ def process_model(
         with tempfile.TemporaryDirectory(dir=OUTPUT_FOLDER) as outdir:
             fp16 = download_base_model(token, model_id, outdir)
             imatrix_path = Path(outdir)/"imatrix.dat"
-            quantized_gguf = quantize_model(outdir, gguf_name, fp16, q_method, use_imatrix, imatrix_q_method, imatrix_path, quant_embedding, embedding_tensor_method, quant_output, output_tensor_method)
+            quantized_gguf = quantize_model(outdir, gguf_name, fp16, q_method, use_imatrix, imatrix_q_method, imatrix_path, quant_embedding, embedding_tensor_method, leave_output, quant_output, output_tensor_method)
 
             # Create empty repo
             api = HfApi(token=token)
@@ -449,9 +453,6 @@ train_data_file = gr.File(
     visible=False
 )
 
-def update_imatrix_visibility(use_imatrix):
-    return gr.update(visible=not use_imatrix), gr.update(visible=use_imatrix), gr.update(visible=use_imatrix)
-
 #####
 # Advanced Options section
 #####
@@ -474,13 +475,16 @@ split_max_size = gr.Textbox(
     visible=False
 )
 
-def update_split_visibility(split_model):
-    return gr.update(visible=split_model), gr.update(visible=split_model)
+leave_output = gr.Checkbox(
+    value=False,
+    label="Leave output tensor",
+    info="Leaves output.weight un(re)quantized"
+)
 
 quant_embedding = gr.Checkbox(
     value=False,
     label="Quant embeddings tensor",
-    info=""
+    info="Quantize embeddings tensor separately"
 )
 embedding_tensor_method = gr.Dropdown(
     choices=["Q2_K", "Q3_K", "Q4_K", "Q5_K", "Q6_K", "Q8_0"],
@@ -494,7 +498,7 @@ embedding_tensor_method = gr.Dropdown(
 quant_output = gr.Checkbox(
     value=False,
     label="Quant output tensor",
-    info=""
+    info="Quantize output tensor separately"
 )
 output_tensor_method = gr.Dropdown(
     choices=["Q2_K", "Q3_K", "Q4_K", "Q5_K", "Q6_K", "Q8_0"],
@@ -505,13 +509,6 @@ output_tensor_method = gr.Dropdown(
     visible=False
 )
 
-def update_embedding_tensor_visibility(quant_embedding):
-    return gr.update(visible=quant_embedding)
-
-def update_output_tensor_visibility(quant_output):
-    return gr.update(visible=quant_output)
-
-
 #####
 # Output Settings section
 #####
@@ -534,10 +531,10 @@ gguf_name = gr.Textbox(
 )
 
 def update_output_repo(model_id, oauth_token: gr.OAuthToken | None):
-    if oauth_token is None or oauth_token.token is None:
+    if oauth_token is None or not oauth_token.token:
         return ""
 
-    if model_id is None:
+    if not model_id:
         return ""
 
     username = whoami(oauth_token.token)["name"]
@@ -545,7 +542,7 @@ def update_output_repo(model_id, oauth_token: gr.OAuthToken | None):
     return f"{username}/{model_name}-GGUF"
 
 def update_output_filename(model_id, use_imatrix, q_method, imatrix_q_method):
-    if model_id is None:
+    if not model_id:
         return ""
 
     model_name = get_model_name(model_id)
@@ -568,6 +565,7 @@ clear_btn = gr.ClearButton(
         imatrix_q_method,
         private_repo,
         train_data_file,
+        leave_output,
         quant_embedding,
         embedding_tensor_method,
         quant_output,
@@ -621,6 +619,7 @@ with gr.Blocks(css=css) as demo:
 
             quant_embedding.render()
             embedding_tensor_method.render()
+            leave_output.render()
             quant_output.render()
             output_tensor_method.render()
 
@@ -661,6 +660,7 @@ with gr.Blocks(css=css) as demo:
             gguf_name,
             quant_embedding,
             embedding_tensor_method,
+            leave_output,
             quant_output,
             output_tensor_method,
             split_model,
@@ -677,27 +677,27 @@ with gr.Blocks(css=css) as demo:
     # OnChange handlers
     #####
     use_imatrix.change(
-        fn=update_imatrix_visibility,
+        fn=lambda use_imatrix: [gr.update(visible=not use_imatrix), gr.update(visible=use_imatrix), gr.update(visible=use_imatrix)],
         inputs=use_imatrix,
         outputs=[q_method, imatrix_q_method, train_data_file]
     )
 
     split_model.change(
-        fn=update_split_visibility,
+        fn=lambda split_model: [gr.update(visible=split_model), gr.update(visible=split_model)],
         inputs=split_model,
         outputs=[split_max_tensors, split_max_size]
     )
 
     quant_embedding.change(
-        fn=update_embedding_tensor_visibility,
+        fn=lambda quant_embedding: gr.update(visible=quant_embedding),
         inputs=quant_embedding,
         outputs=[embedding_tensor_method]
     )
 
     quant_output.change(
-        fn=update_output_tensor_visibility,
+        fn=lambda quant_output: [gr.update(visible=quant_output), gr.update(visible=not quant_output)],
         inputs=quant_output,
-        outputs=[output_tensor_method]
+        outputs=[output_tensor_method, leave_output]
     )
 
     model_id.change(