I'm trying to automate data retrieval from a portal using Selenium and Chrome WebDriver. The data includes Insurance Company name, Patient name, Claim number, claimed and approved amount, Status, date of admission, and date of discharge. My code works fine locally but encounters issues when deployed on a cloud server.
I used Selenium and chrome WebDriver to achieve the appropriate output. Environment, Files location, WebDriver compatibility, Permissions all are same at local and server both sides. My front end code in Angular, backend code in C#. From C# I called a python script. So after calling python script via button I want to fetch all table values in my database.
import osimport mysql.connectorfrom selenium import webdriverfrom selenium.webdriver.common.by import Byfrom selenium.webdriver.firefox.options import Optionsfrom selenium.webdriver.support.ui import WebDriverWaitfrom selenium.webdriver.support import expected_conditions as ECimport pandas as pdimport time# Variable Declarationusername = "ABCD"password = "*****"# URL Of The Login Pagelogin_url = "############"table_url = '$$$$$$$$$$$'# Set up GeckoDriver pathgeckodriver_path = r'C:\WebDrivers\chromedriver.exe'os.environ['PATH'] += ';'+ os.path.dirname(geckodriver_path)# Set up Firefox optionsoptions = Options()options.add_argument("--headless") # Run Firefox in headless mode if needed# Initialize WebDriver with optionsdriver = webdriver.Firefox(options=options)try: # Open the login page driver.get(login_url) # Wait for the username field to be present and enter the username WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, 'login-form_username'))).send_keys(username) # Wait for the password field to be present and enter the password WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.ID, 'login-form_password'))).send_keys(password) # Wait for the login button to be clickable and click it login_button = WebDriverWait(driver, 10).until( EC.element_to_be_clickable((By.XPATH, "//*[@id='login-form']/div[4]/div/button/span")) ) login_button.click() # Wait for the navigation to the claims table page WebDriverWait(driver, 20).until(EC.url_to_be(table_url)) time.sleep(5) # Additional delay to ensure data loads completely # Extracting table data directly using Selenium table = WebDriverWait(driver, 10).until(EC.presence_of_element_located((By.TAG_NAME, 'table'))) rows = table.find_elements(By.TAG_NAME, 'tr') data = [] for row in rows: cols = row.find_elements(By.TAG_NAME, 'td') if cols: # Avoids headers or empty rows data.append([col.text for col in cols]) # Convert list to DataFrame df = pd.DataFrame(data, columns=['InsuranceCompanyName', 'ClaimNumber', 'PatientName', 'ClaimedAmount', 'ApprovedAmount', 'Status', 'DateOfAdmission', 'DateOfDischarge', 'Portal']) # Connect to MySQL database conn = mysql.connector.connect(user='##', password='##', host='##', database='##') cursor = conn.cursor() # Insert data into MySQL table for _, row in df.iterrows(): cursor.execute(""" INSERT INTO Claim (InsuranceCompanyName, ClaimNumber, PatientName, ClaimedAmount, ApprovedAmount, Status, DateOfAdmission, DateOfDischarge, Portal) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)""", tuple(row)) # Commit changes and close connection conn.commit() cursor.close() conn.close()finally: # Close the WebDriver driver.quit()# Optionally, print or inspect the DataFrameprint(df)
I facing the following error after clicked frontend button,
File "C:\Python310\lib\site-packages\selenium\webdriver\common\driver_finder.py", line 38, in get_path path = SeleniumManager().driver_location(options) if path is None else pathFile "C:\Python310\lib\site-packages\selenium\webdriver\common\selenium_manager.py", line 106, in driver_location output = self.run(args)File "C:\Python310\lib\site-packages\selenium\webdriver\common\selenium_manager.py", line 154, in run raise WebDriverException(f"Unsuccessful command executed: {command}.\n{result}{stderr}")selenium.common.exceptions.WebDriverException: Message: Unsuccessful command executed: C:\Python310\lib\site-packages\selenium\webdriver\common\windows\selenium-manager.exe --browser firefox --language-binding python --output json.{'code': 65, 'message': 'Access is denied. (os error 5)', 'driver_path': '', 'browser_path': ''}