# -*- coding: utf-8 -*-
from selenium import webdriver
from selenium.webdriver.firefox.options import Options
import requests
from datetime import datetime
from dateutil import relativedelta
import re
import os
import whois
import csv
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import Select

"""
    Query whois data
    Return whois data or error if unable to acquire
"""
def whois_info(domain, date_data):

    if(domain.endswith("/")):
        domain = domain[:-1]

    if('http' in domain):
        domain = re.sub(r'http://|https://','',domain)

    try:  
        date_diff = "Unable to calculate difference."
        wresults = whois.query(domain)
        if wresults is not None:
            date_diff = calc_creation_diff(str(wresults.creation_date).split(" ")[0],date_data)  
            return "Name: " + wresults.name+"\n" + "Emails: " + ' '.join(wresults.emails)+ '\n' + "Creation Date: " + str(wresults.creation_date) +'\n' + "Last Updated: " + str(wresults.last_updated) +'\n' + "Expiration Date: " +  str(wresults.expiration_date) +'\n' + "Name Servers: " +' '.join(wresults.name_servers), domain + " - " + date_diff
        else:
            return "Unable to acquire whois data for "+domain, date_diff
    except Exception as ex:
        print("Whois - Error: " + str(ex))
        return "Unable to acquire whois data for " + domain, date_diff

"""
    Check domain status
    Format query to string
    Returns domain status string
"""
def domain_status(domain):
    status_result = ""

    try:

        if 'http' not in domain.lower():
            domain = 'https://' + domain.strip()
        status = requests.head(domain.strip()).status_code
        if status == 200:
            print('Alive as of: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
            status_result = 'Alive as of: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
        elif status == 404:
            print('Does not exist as of: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
            status_result = 'Does not exist as of: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
        elif status == 503 or status == 500:
            print('Unavailable')
            status_result = 'Unavailable as of: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
        else:
            print('Unknown status:' + str(status) + " as of: " + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
            status_result = 'Unknown status:' + str(status) + " as of: " + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
    except requests.exceptions.ConnectionError:
        return domain + " - Connection Error"
    except Exception as ex: 
        return domain + ' - Unable to get status of URL: ' + str(ex)

    return domain + " - " + status_result

"""
    Calculate difference in domain creation and article date
    Returns difference of year/month/day in string
"""
def calc_creation_diff(domain_creation_date,article_date):

    formatted_article_date = datetime.strptime(article_date,"%d %b %Y")
    formatted_domain_creation_date = datetime.strptime(domain_creation_date,"%Y-%m-%d")
    date_diff = relativedelta.relativedelta(formatted_article_date,formatted_domain_creation_date)
    return str(date_diff.years) + " Years " + str(date_diff.months) + " Months " + str(date_diff.days) + " Days"

def write_csv_data(data):

    csv_file_name = 'iosco_crawled_data_selenium.csv'
    file_exist = os.path.isfile(csv_file_name)
    with open(csv_file_name, 'a') as csvfile:

        headers = ['Date','Title','Company','Regulator','Jurisdiction','Domain','Subject','Comments', 'Website Status', 'Whois Information', 'Report to Creation Timeframe']
        writer = csv.writer(csvfile, delimiter=',')

        if (not file_exist):
            writer.writerow(headers)

        writer.writerow(data)

"""
    Get article links
    Returns list of found article links
"""
def get_links():
    try:
        all_links = []
        options = Options()
        options.add_argument("--headless")
        driver = webdriver.Firefox(options=options)
        base_url = "https://www.iosco.org/investor_protection/?subsection=investor_alerts_portal"
        driver.get(base_url)
        filter_btn = driver.find_element(By.ID,'btnSubmit')
        select_box = driver.find_element(By.ID,"JurisdictionID") 
        jurisdiction_list = Select(select_box)
        jurisdiction_list = [x.text for x in select_box.find_elements(By.TAG_NAME,"option")]
        for jur in jurisdiction_list[1:]:
            select_box = driver.find_element(By.ID,"JurisdictionID") 
            filter_btn = driver.find_element(By.ID,'btnSubmit')

            select_jur = Select(select_box)
            select_jur.select_by_visible_text(jur)
            # print(jur)
            filter_btn.click()
            tab = driver.find_element(By.CLASS_NAME, 'defaultStyle')
            tr = tab.find_elements(By.TAG_NAME, 'tr')
            for row in tr:

                check_a = row.find_elements(By.TAG_NAME, 'a')
                if (check_a):
                    row_val = row.find_element(By.TAG_NAME, 'a')
                    href = row_val.get_attribute('href')
                    if('&id=' in href):
                        all_links.append(href)

        driver.quit()
        return all_links
    except Exception as ex:
        print(ex)
    
def main():

    links = get_links()
    for link in links:

        whois_list = []
        status_list = []
        date_diff_list = []
        options = Options()
        options.add_argument("--headless")
        driver = webdriver.Firefox(options=options)
        print(link)
        driver.get(link)
        # Get article text
        page_data = driver.find_elements(By.CLASS_NAME,"description")
        title = driver.find_element(By.TAG_NAME, 'h1').text
        company = page_data[0].text
        regulator = page_data[1].text
        jurisdiction = page_data[2].text
        date = page_data[3].text
        link = page_data[4].text
        subject = page_data[5].text
        comments = page_data[6].text
        driver.quit()
        # Regex to check domain
        domain_regex = re.compile(r'(?:https?)?(?::\/\/)?[\w\.-]+\.(?:[a-z]+)(?:\/[-\w\.\/]*)*', re.IGNORECASE)
        # Iterate through found domains and check domain data
        for domain in re.findall(domain_regex,company):
            whois_results, date_diff = whois_info(domain.strip(),date)
            date_diff_list.append(date_diff)
            whois_list.append(whois_results)
            status_list.append(domain_status(domain.strip()))
        status_data = '\n'.join(status_list)
        whois_data = '\n'.join(whois_list)
        date_diff_data = '\n'.join(date_diff_list)
        csv_row_data = [date,"\""+title+"\"",company,regulator,jurisdiction,link,subject,comments,status_data,whois_data,date_diff_data]
        
        write_csv_data(csv_row_data)

if __name__ == "__main__":
    main()