
# Script for reading data from Arduino (via USB), then storing them into CSV file and sending the file to the server
# v1, 19.4.2024, VUT FEKT Brno, Vaculík Samuel

# --------------------------------------------------
#                     LIBRARIES
# --------------------------------------------------

import sys
import os
import serial
import csv
import time
import json
import threading
import gzip
import shutil
import requests
from requests.auth import HTTPBasicAuth
from requests.exceptions import HTTPError, ConnectionError, Timeout, RequestException
from dotenv import load_dotenv
from queue import Queue


# --------------------------------------------------
#                     VARIABLES
# --------------------------------------------------

SERVER_SENDING_PERIOD = 10     # In seconds
BAUD_RATE = 115200             # Transfer speed of the serial communication
ser = None                     # Python object for the serial communication
SERIAL_PORT = '/dev/ttyACM0'   # Name of the serial port on which the Arduino is connected


# --------------------------------------------------
#                    DEFINITIONS
# --------------------------------------------------

# Create CSV file name
file_name = sys.argv[1]      # Get the name from the main.py script
csv_file = f'/home/pi/BP_Vaculik/Logs/Arduino/Original/arduino_{file_name}.csv'
compressed_csv_file = f'/home/pi/BP_Vaculik/Logs/Arduino/Compressed/arduino_{file_name}.gz'

# Headers in CSV file
headers = [
           'Timestamp',
           'Acc1_X', 'Acc1_Y', 'Acc1_Z',
           'Acc2_X', 'Acc2_Y', 'Acc2_Z',
           'Motor_temp_X', 'Motor_temp_Y', 'Motor_temp_E', 'Motor_temp_J',
           'Motor_temp_Z1', 'Motor_temp_Z2', 'Motor_temp_Z3', 'Motor_temp_Z4',
           'YZC_Weight', 'Temp_Bottom', 'Temp_Top', 'Hum_Bottom', 'Hum_Top',
           'MQ135_PPM', 'MQ7_PPM'
           ]

# Object for storing data
data_queue = Queue()

# Server url addresses (Only test server is used at this moment)
url_post = 'https://httpbin.org/post'
url_auth = 'https://httpbin.org/basic-auth/user/passwd'

# Get login data to the server from the '.env' file
dotenv_path = '/home/pi/BP_Vaculik/.env'
load_dotenv(dotenv_path)
server_username = os.getenv('SERVER_USERNAME')
server_password = os.getenv('SERVER_PASSWORD')

# Try to start serial communication. The maximum number of attempts is 10.
for try_number in range(1, 11):
    try:
        ser = serial.Serial(SERIAL_PORT, BAUD_RATE, timeout=1)
        ser.flushInput()
        time.sleep(1)
        print('The serial port has been successfully connected.')
        break
    except serial.SerialException as e:
        print(f'{try_number}. Error when opening the serial port: {e}')
        if try_number < 10:
            time.sleep(1)
            break
        else:
            print('Unable to open serial port! The program will be terminated.')
            exit(1)

# Try to authenticate to the server address. The maximum number of attempts is 10.
for try_number in range(1, 11):

    try:
        response_auth = requests.get(url_auth, auth=HTTPBasicAuth(server_username, server_password))
        response_auth.raise_for_status()

        if response_auth.status_code == 200:
            print('Authentication was successful.')
            break

    except Exception as e:
        print(f'{try_number}. Error during authentication: {e}')
        if try_number < 10:
            time.sleep(1)
        else:
            print('Can not authenticate to the server! The program will be terminated.')
            exit(2)


# --------------------------------------------------
#                     FUNCTIONS
# --------------------------------------------------

# Function for sending the compressed CSV file to the server
def send_file():
    print("Sending to server...")

    try:
        with open(compressed_csv_file, 'rb') as file:
            # Prepare compressed file for sending to the server (key/name, actual file, format)
            file_for_send = {'file': (compressed_csv_file.split('/')[-1], file, 'application/gzip')}

            try:
                # Actual sending to the server
                response = requests.post(url_post, files=file_for_send, timeout=5)
                response.raise_for_status()
                print("The compressed file has been successfully sent to the server.")

            except HTTPError as e:
                print(f'HTTP error when sending a request: {e}')
                pass
            except ConnectionError as e:
                print(f'Connection error: {e}')
                pass
            except Timeout as e:
                print(f'Request timeout has expired: {e}')
                pass
            except RequestException as e:
                print(f'Error when sending a request: {e}')
                pass
            except Exception as e:
                print(f'Unexpected error: {e}')
                pass

    except FileNotFoundError as e:
        print(f'The file {compressed_csv_file} does not exist: {e}')
    except TypeError as e:
        print(f'Something is wrong with the file {compressed_csv_file}: {e}')
    except OSError as e:
        print(f'Operating system error: {e}')


# Function for compressing the CSV file
def compress_file():
    try:
        with open(csv_file, 'rb') as file_original:
            try:
                with gzip.open(compressed_csv_file, 'wb') as file_compressed:
                    # Copying data between files, data is also compressed at the same time
                    shutil.copyfileobj(file_original, file_compressed)

            except TypeError as e:
                print(f'Something is wrong with the file {compressed_csv_file}: {e}')
                pass
            except PermissionError:
                print(f"You do not have permission to write into the file {compressed_csv_file}!")
                pass
            except IsADirectoryError:
                print(f"The specified path ti the file is a directory! The path is: {compressed_csv_file}")
                pass
            except OSError as e:
                print(f"Operating system error: {e}")
                pass

    except FileNotFoundError as e:
        print(f'File {csv_file} does not exist: {e}')
    except TypeError as e:
        print(f'Something is wrong with the file {csv_file}: {e}')
        pass
    except PermissionError:
        print(f"You do not have permission to read the file {csv_file}!")
        pass
    except IsADirectoryError:
        print(f"The specified path to the file is a directory! The path is: {csv_file}")
        pass
    except OSError as e:
        print(f"Operating system error: {e}")
        pass


# Function to transfer data from queue to the CSV file
def process_data_in_queue():
    try:
        with open(csv_file, 'w', newline='') as csvfile:
            writer = csv.DictWriter(csvfile, fieldnames=headers)            # Create object for writing data
            writer.writeheader()                                            # Insert headers into the CSV file
            while not data_queue.empty():                                   # Emptying the queue into the CSV file
                writer.writerow(data_queue.get())

    except TypeError as e:
        print(f'Something is wrong with the file {csv_file}: {e}')
        pass
    except ValueError:
        print(f"The file contains keys that are not in the 'fieldnames' list!")
        pass
    except PermissionError:
        print(f"You do not have permission to write into the file {csv_file}!")
        pass
    except IsADirectoryError:
        print(f"The specified path to the file is a directory! Path: {csv_file}")
        pass
    except OSError as e:
        print(f"Operating system error: {e}")
        pass


# Row creation function for CSV file
def store_data(data):
    row = {
        'Timestamp':     int((time.time() - start_time) * 1000),           # Get the elapsed time in milliseconds
        'Acc1_X':        data.get('Acc1', {}).get('X', ''),                # Put blank character if any data is missing
        'Acc1_Y':        data.get('Acc1', {}).get('Y', ''),
        'Acc1_Z':        data.get('Acc1', {}).get('Z', ''),
        'Acc2_X':        data.get('Acc2', {}).get('X', ''),
        'Acc2_Y':        data.get('Acc2', {}).get('Y', ''),
        'Acc2_Z':        data.get('Acc2', {}).get('Z', ''),
        'Motor_temp_X':  data.get('Motors_Temp', {}).get('X', ''),
        'Motor_temp_Y':  data.get('Motors_Temp', {}).get('Y', ''),
        'Motor_temp_E':  data.get('Motors_Temp', {}).get('E', ''),
        'Motor_temp_J':  data.get('Motors_Temp', {}).get('J', ''),
        'Motor_temp_Z1': data.get('Motors_Temp', {}).get('Z1', ''),
        'Motor_temp_Z2': data.get('Motors_Temp', {}).get('Z2', ''),
        'Motor_temp_Z3': data.get('Motors_Temp', {}).get('Z3', ''),
        'Motor_temp_Z4': data.get('Motors_Temp', {}).get('Z4', ''),
        'YZC_Weight':    data.get('YZC_Weight', ''),
        'Temp_Bottom':   data.get('Temp', {}).get('Bottom', ''),
        'Temp_Top':      data.get('Temp', {}).get('Top', ''),
        'Hum_Bottom':    data.get('Hum', {}).get('Bottom', ''),
        'Hum_Top':       data.get('Hum', {}).get('Top', ''),
        'MQ135_PPM':     data.get('MQ135_PPM', ''),
        'MQ7_PPM':       data.get('MQ135_PPM', ''),
        }

    data_queue.put(row)                                                    # Insert the created row into the queue


# --------------------------------------------------
#                THREAD FUNCTION
# --------------------------------------------------

def periodic_sending_to_the_server():
    while True:
        time.sleep(SERVER_SENDING_PERIOD)
        process_data_in_queue()
        compress_file()
        send_file()


# --------------------------------------------------
#                     MAIN
# --------------------------------------------------

def main():
    while True:
        if ser.is_open and ser.in_waiting > 0:                          # If serial bus contains new data
            try:
                line = ser.readline().decode('utf-8').rstrip()          # Read whole message from Arduino

                try:
                    arduino_data = json.loads(line)                     # Convert JSON string to Python dictionary
                    store_data(arduino_data)                            # Store message into queue

                except json.JSONDecodeError:
                    print(f"Error when decoding JSON message: {line}")
                    pass

            except serial.SerialTimeoutException:
                print("Reading from the serial port exceeded the time limit!")
                pass
            except serial.SerialException as e:
                print(f"Error when reading from serial port: {e}")
                pass
            except UnicodeDecodeError as e:
                print("Error when decoding message using UTF-8!")
                pass


# --------------------------------------------------
#                     STARTUP
# --------------------------------------------------

if __name__ == '__main__':
    # Store program start time for timestamps
    start_time = time.time()

    # Sets up an independent loop for sending data to the server
    threading.Thread(target=periodic_sending_to_the_server, daemon=True).start()

    # Actual start of the script
    main()
