added error trapping when no data was received

Layoutlmv3_inference/ocr.py

@@ -72,7 +72,229 @@ def enhance_txt(img, intensity_increase=20, bilateral_filter_diameter=9, bilater
     img = np.clip(img + intensity_increase, 0, 255).astype(np.uint8)
 
     # Apply bilateral filter to reduce noise
-    img = cv2.bilateralFilter(img, bilateral_filter_diameter, bilateral_filter_sigma_color, bilateral_filter_sigma_space)
+    img = cv2.bilateralFilter(img, bilaterimport os
+import pandas as pd
+import cv2
+import numpy as np
+import json
+import requests
+import traceback
+import tempfile
+from rembg import remove
+
+
+from PIL import Image
+
+
+def preprocess_image(image_path, max_file_size_mb=1, target_file_size_mb=0.5):
+    try:
+        # Read the image
+        image = cv2.imread(image_path)
+        # Enhance text
+        enhanced = enhance_txt(image)
+
+        # Save the enhanced image to a temporary file
+        temp_file_path = tempfile.NamedTemporaryFile(suffix='.jpg').name
+        cv2.imwrite(temp_file_path, enhanced)
+
+        # Check file size of the temporary file
+        file_size_mb = os.path.getsize(
+            temp_file_path) / (1024 * 1024)  # Convert to megabytes
+
+        while file_size_mb > max_file_size_mb:
+            print(
+                f"File size ({file_size_mb} MB) exceeds the maximum allowed size ({max_file_size_mb} MB). Resizing the image.")
+            ratio = np.sqrt(target_file_size_mb / file_size_mb)
+            new_width = int(image.shape[1] * ratio)
+            new_height = int(image.shape[0] * ratio)
+
+            # Resize the image
+            enhanced = cv2.resize(enhanced, (new_width, new_height))
+
+            # Save the resized image to a temporary file
+            temp_file_path = tempfile.NamedTemporaryFile(suffix='.jpg').name
+            cv2.imwrite(temp_file_path, enhanced)
+
+            # Update file size
+            file_size_mb = os.path.getsize(temp_file_path) / (1024 * 1024)
+            print(f"New file size: ({file_size_mb} MB)")
+
+        # Return the final resized image
+        image_resized = cv2.imread(temp_file_path)
+        return image_resized
+
+    except Exception as e:
+        print(f"An error occurred in preprocess_image: {str(e)}")
+        return None
+
+
+def enhance_txt(img, intensity_increase=20, bilateral_filter_diameter=9, bilateral_filter_sigma_color=75, bilateral_filter_sigma_space=75):
+    # Get the width and height of the image
+    w = img.shape[1]
+    h = img.shape[0]
+    w1 = int(w * 0.05)
+    w2 = int(w * 0.95)
+    h1 = int(h * 0.05)
+    h2 = int(h * 0.95)
+    ROI = img[h1:h2, w1:w2]  # 95% of the center of the image
+    threshold = np.mean(ROI) * 0.88  # % of average brightness
+
+    # Convert image to grayscale
+    grayscale_img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
+
+    # Find contours
+    contours, _ = cv2.findContours(
+        grayscale_img, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+
+    # # Apply Gaussian blur
+    blurred = cv2.GaussianBlur(grayscale_img, (1, 1), 0)
+
+    edged = 255 - cv2.Canny(blurred, 100, 150, apertureSize=7)
+
+    # Increase intensity by adding a constant value
+    img = np.clip(img + intensity_increase, 0, 255).astype(np.uint8)
+
+    # Apply bilateral filter to reduce noise
+    img = cv2.bilateralFilter(img, bilateral_filter_diameter,
+                              bilateral_filter_sigma_color, bilateral_filter_sigma_space)
+
+    _, binary = cv2.threshold(blurred, threshold, 255, cv2.THRESH_BINARY)
+    return binary
+
+
+def run_tesseract_on_preprocessed_image(preprocessed_image, image_path):
+    image_name = os.path.basename(image_path)
+    image_name = image_name[:image_name.find('.')]
+
+    # Create the "temp" folder if it doesn't exist
+    temp_folder = "static/temp"
+    if not os.path.exists(temp_folder):
+        os.makedirs(temp_folder)
+
+    # Define the OCR API endpoint
+    url = "https://api.ocr.space/parse/image"
+
+    # Define the API key and the language
+    api_key = os.getenv("ocr_space")
+    language = "eng"
+
+    # Save the preprocessed image
+    cv2.imwrite(os.path.join(
+        temp_folder, f"{image_name}_preprocessed.jpg"), preprocessed_image)
+
+    # Open the preprocessed image file as binary
+    with open(os.path.join(temp_folder, f"{image_name}_preprocessed.jpg"), "rb") as f:
+        # Define the payload for the API request
+        payload = {
+            "apikey": api_key,
+            "language": language,
+            "isOverlayRequired": True,
+            "OCREngine": 2
+        }
+        # Define the file parameter for the API request
+        file = {
+            "file": f
+        }
+        # Send the POST request to the OCR API
+        response = requests.post(url, data=payload, files=file)
+
+
+        # Check the status code of the response
+        if response.status_code == 200:
+            # Parse the JSON response
+            result = response.json()
+            print("---JSON file saved")
+            # Save the OCR result as JSON
+            with open(os.path.join(temp_folder, f"{image_name}_ocr.json"), 'w') as f:
+                json.dump(result, f)
+
+            return os.path.join(temp_folder, f"{image_name}_ocr.json")
+        else:
+            raise Exception("An error occurred: " + response.text)
+
+
+def clean_tesseract_output(json_output_path):
+    try:
+        with open(json_output_path, 'r') as json_file:
+            data = json.load(json_file)
+
+        lines = data['ParsedResults'][0]['TextOverlay']['Lines']
+
+        words = []
+        for line in lines:
+            for word_info in line['Words']:
+                word = {}
+                origin_box = [
+                    word_info['Left'],
+                    word_info['Top'],
+                    word_info['Left'] + word_info['Width'],
+                    word_info['Top'] + word_info['Height']
+                ]
+
+                word['word_text'] = word_info['WordText']
+                word['word_box'] = origin_box
+                words.append(word)
+
+        return words
+    except (KeyError, IndexError, FileNotFoundError, json.JSONDecodeError) as e:
+        print(f"Check your Internet Connection.")
+
+        print(f"Error cleaning Tesseract output: {str(e)}")
+        return None
+
+
+def prepare_batch_for_inference(image_paths):
+    # print("my_function was called")
+    # traceback.print_stack()  # This will print the stack trace
+    # Print the total number of images to be processed
+    print(f"Number of images to process: {len(image_paths)}")
+    print("1. Preparing for Inference")
+    tsv_output_paths = []
+
+    inference_batch = dict()
+    print("2. Starting Preprocessing")
+    # Ensure that the image is only 1
+    for image_path in image_paths:
+        # Print the image being processed
+        print(f"Processing the image: {image_path}")
+        print("3. Preprocessing the Receipt")
+        preprocessed_image = preprocess_image(image_path)
+        if preprocessed_image is not None:
+            try:
+                print("4. Preprocessing done. Running OCR")
+                try:
+                    json_output_path = run_tesseract_on_preprocessed_image(
+                        preprocessed_image, image_path)
+                except Exception as e:
+                    print(f"An error has occured: {str(e)}")
+                    raise e
+                print("5. OCR Complete")
+            except Exception as e:
+                print(f"An error has occured: {str(e)}")
+                raise e
+            if json_output_path:
+                tsv_output_paths.append(json_output_path)
+
+    print("6. Preprocessing and OCR Done")
+    # clean_outputs is a list of lists
+    clean_outputs = [clean_tesseract_output(
+        tsv_path) for tsv_path in tsv_output_paths]
+    print("7. Cleaned OCR output")
+    word_lists = [[word['word_text'] for word in clean_output]
+                  for clean_output in clean_outputs]
+    print("8. Word List Created")
+    boxes_lists = [[word['word_box'] for word in clean_output]
+                   for clean_output in clean_outputs]
+    print("9. Box List Created")
+    inference_batch = {
+        "image_path": image_paths,
+        "bboxes": boxes_lists,
+        "words": word_lists
+    }
+
+    print("10. Prepared for Inference Batch")
+    return inference_batch
+al_filter_diameter, bilateral_filter_sigma_color, bilateral_filter_sigma_space)
 
     _, binary = cv2.threshold(blurred, threshold, 255, cv2.THRESH_BINARY)
     return binary

app.py

@@ -24,7 +24,6 @@ import signal
 import shutil
 from datetime import datetime
 import zipfile
-from pathlib import Path
 
 # LLM
 import argparse
@@ -34,10 +33,17 @@ from Layoutlmv3_inference.inference_handler import handle
 import logging
 import os
 import copy
+import warnings
+warnings.filterwarnings("ignore", category=UserWarning, module='torch.serialization', lineno=1113)
+warnings.filterwarnings("ignore")
+from torch.serialization import SourceChangeWarning
+
+warnings.filterwarnings("ignore", category=FutureWarning)
+warnings.filterwarnings("ignore", category=SourceChangeWarning)
 
 
 # Upload Folder
-UPLOAD_FOLDER = r'static/temp/uploads'
+UPLOAD_FOLDER = 'static/temp/uploads'
 if not os.path.exists(UPLOAD_FOLDER):
     os.makedirs(UPLOAD_FOLDER)
 
@@ -83,7 +89,7 @@ def allowed_file(filename):
 
 @app.route('/upload', methods=['GET', 'POST'])
 def upload_files():
-    UPLOAD_FOLDER = r'static/temp/uploads'
+    UPLOAD_FOLDER = 'static/temp/uploads'
     if not os.path.exists(UPLOAD_FOLDER):
         os.makedirs(UPLOAD_FOLDER)
     if request.method == 'POST':
@@ -102,13 +108,12 @@ def upload_files():
     return render_template('index.html')
 
 
-from pathlib import Path
 def make_predictions(image_paths):
-    # temp = None
+    temp = None
     try:
-        # # For Windows OS
-        # temp = pathlib.PosixPath  # Save the original state
-        # pathlib.PosixPath = pathlib.WindowsPath  # Change to WindowsPath temporarily
+        # For Windows OS
+        temp = pathlib.PosixPath  # Save the original state
+        pathlib.PosixPath = pathlib.WindowsPath  # Change to WindowsPath temporarily
         
         model_path = Path(r'model/export')
         learner = load_learner(model_path)
@@ -126,16 +131,14 @@ def make_predictions(image_paths):
             predicted_class_str = str(prediction_class)
             
             predictions.append(predicted_class_str)
-            
-            print(f"Prediction: {predictions}")
 
         return predictions
 
     except Exception as e:
         return {"error in make_predictions": str(e)}
         
-    # finally:
-    #     pathlib.PosixPath = temp 
+    finally:
+        pathlib.PosixPath = temp 
 
 
 @app.route('/predict/<filenames>', methods=['GET', 'POST'])
@@ -168,12 +171,8 @@ def predict_files(filenames):
         
         if os.path.exists(file_path):
             # Call make_predictions automatically
-            prediction_result = make_predictions([file_path])
-            if isinstance(prediction_result, list) and len(prediction_result) > 0:
-                prediction_results.append(prediction_result[0])  # Append only the first prediction result
-            else:
-                print(f"Error making prediction for {file}: {prediction_result}")
-
+            prediction_result = make_predictions([file_path])  # Pass file_path as a list
+            prediction_results.append(prediction_result[0])  # Append only the first prediction result
             prediction_results_copy = copy.deepcopy(prediction_results)
 
             non_receipt_indices = []
@@ -187,15 +186,16 @@ def predict_files(filenames):
                 if os.path.exists(file_to_remove):
                     os.remove(file_to_remove)
                     
+
     return render_template('extractor.html', index_url=index_url, image_paths=image_paths, prediction_results = prediction_results, predictions=dict(zip(image_paths, prediction_results_copy)))
 
     
     
-# @app.route('/get_inference_image')
-# def get_inference_image():
-#     # Assuming the new image is stored in the 'inferenced' folder with the name 'temp_inference.jpg'
-#     inferenced_image = 'static/temp/inferenced/temp_inference.jpg'
-#     return jsonify(updatedImagePath=inferenced_image), 200  # Return the image path with a 200 status code
+@app.route('/get_inference_image')
+def get_inference_image():
+    # Assuming the new image is stored in the 'inferenced' folder with the name 'temp_inference.jpg'
+    inferenced_image = 'static/temp/inferenced/temp_inference.jpg'
+    return jsonify(updatedImagePath=inferenced_image), 200  # Return the image path with a 200 status code
     
 
 def process_images(model_path: str, images_path: str) -> None:
@@ -205,12 +205,14 @@ def process_images(model_path: str, images_path: str) -> None:
         inference_batch = prepare_batch_for_inference(images_path)
         context = {"model_dir": model_path}
         handle(inference_batch, context)
-    except Exception as err:
+    except Exception as e:
+        print("No Internet connection.")
         os.makedirs('log', exist_ok=True)
         logging.basicConfig(filename='log/error_output.log', level=logging.ERROR,
                             format='%(asctime)s %(levelname)s %(name)s %(message)s')
         logger = logging.getLogger(__name__)
         logger.error(err)
+        return redirect(url_for('index'))
 
 @app.route('/run_inference', methods=['GET'])
 def run_inference():
@@ -335,24 +337,8 @@ def create_csv():
 
     except Exception as e:
         print(f"An error occurred in create_csv: {str(e)}")
-        return None
+        return render_template('extractor.html', error_message=str(e))
 
-    except Exception as e:
-        print(f"An error occurred in create_csv: {str(e)}")
-        return None
-
-    except FileNotFoundError as e:
-        print(f"File not found error: {str(e)}")
-        return jsonify({'error': 'File not found.'}), 404
-    except json.JSONDecodeError as e:
-        print(f"JSON decoding error: {str(e)}")
-        return jsonify({'error': 'JSON decoding error.'}), 500
-    except csv.Error as e:
-        print(f"CSV error: {str(e)}")
-        return jsonify({'error': 'CSV error.'}), 500
-    except Exception as e:
-        print(f"An unexpected error occurred: {str(e)}")
-        return jsonify({'error': 'An unexpected error occurred.'}), 500
         
 @app.route('/get_data')
 def get_data():
@@ -372,6 +358,5 @@ def download_csv():
         return jsonify({"error": f"Download failed: {str(e)}"})
 
 
-
 if __name__ == '__main__':
     app.run(debug=True)

templates/extractor.html

@@ -223,6 +223,10 @@
 
 
                 function updateTable(data) {
+                    if (!data || data.trim() === '') {
+                        return 'Error extracting the information: data is empty';
+                    }
+
                     Papa.parse(data, {
                         header: true,
                         skipEmptyLines: true,
@@ -230,37 +234,60 @@
                             const tbody = document.querySelector('#dataTable tbody');
                             tbody.innerHTML = ''; // Clear existing rows
 
-                            results.data.forEach(row => {
-                                const RECEIPTNUMBER = row['RECEIPTNUMBER'] || '';
-                                const MERCHANTNAME = row['MERCHANTNAME'] || '';
-                                const MERCHANTADDRESS = row['MERCHANTADDRESS'] || '';
-                                const TRANSACTIONDATE = row['TRANSACTIONDATE'] || '';
-                                const TRANSACTIONTIME = row['TRANSACTIONTIME'] || '';
-                                const ITEMS = row['ITEMS'] || '';
-                                const PRICE = row['PRICE'] || '';
-                                const TOTAL = row['TOTAL'] || '';
-                                const VATTAX = row['VATTAX'] || '';
-
+                            if (results.data.length === 0) {
                                 const tr = document.createElement('tr');
-                                tr.innerHTML = `
-                    <td contenteditable="true">${RECEIPTNUMBER}</td>
-                    <td contenteditable="true">${MERCHANTNAME}</td>
-                    <td contenteditable="true">${MERCHANTADDRESS}</td>
-                    <td contenteditable="true">${TRANSACTIONDATE}</td>
-                    <td contenteditable="true">${TRANSACTIONTIME}</td>
-                    <td contenteditable="true">${ITEMS}</td>
-                    <td contenteditable="true">${PRICE}</td>
-                    <td contenteditable="true">${TOTAL}</td>
-                    <td contenteditable="true">${VATTAX}</td>
-                `;
+                                const td = document.createElement('td');
+                                td.colSpan = 9;
+                                td.style.textAlign = 'center'; // Center the content
+                                td.style.fontSize = '1.2em'; // Increase the font size
+                                td.style.height = '100px'; // Set the height of the row
+                                td.style.verticalAlign = 'middle'; // Vertically center the content
+
+                                const icon = document.createElement('i');
+                                icon.className = 'fas fa-exclamation-triangle'; // Replace with the class of your error icon
+                                td.appendChild(icon);
+                                
+                                const br = document.createElement('br'); // Create a line break element
+                                td.appendChild(br); // Append the line break element
+
+                                const text = document.createTextNode(' An error occurred in extracting the data. Please contact the developers.');
+                                td.appendChild(text);
+
+                                tr.appendChild(td);
                                 tbody.appendChild(tr);
-                            });
+                            } else {
+                                results.data.forEach(row => {
+                                    const RECEIPTNUMBER = row['RECEIPTNUMBER'] || '';
+                                    const MERCHANTNAME = row['MERCHANTNAME'] || '';
+                                    const MERCHANTADDRESS = row['MERCHANTADDRESS'] || '';
+                                    const TRANSACTIONDATE = row['TRANSACTIONDATE'] || '';
+                                    const TRANSACTIONTIME = row['TRANSACTIONTIME'] || '';
+                                    const ITEMS = row['ITEMS'] || '';
+                                    const PRICE = row['PRICE'] || '';
+                                    const TOTAL = row['TOTAL'] || '';
+                                    const VATTAX = row['VATTAX'] || '';
+
+                                    const tr = document.createElement('tr');
+                                    tr.innerHTML = `
+                                        <td contenteditable="true">${RECEIPTNUMBER}</td>
+                                        <td contenteditable="true">${MERCHANTNAME}</td>
+                                        <td contenteditable="true">${MERCHANTADDRESS}</td>
+                                        <td contenteditable="true">${TRANSACTIONDATE}</td>
+                                        <td contenteditable="true">${TRANSACTIONTIME}</td>
+                                        <td contenteditable="true">${ITEMS}</td>
+                                        <td contenteditable="true">${PRICE}</td>
+                                        <td contenteditable="true">${TOTAL}</td>
+                                        <td contenteditable="true">${VATTAX}</td>
+                                    `;
+                                    tbody.appendChild(tr);
+                                });
+                            }
                         }
                     });
                 }
-
             });
 
+
             document.querySelector('#downloadButton').addEventListener('click', function (event) {
                 event.preventDefault();  // Prevent the default action