
import re
import os
import csv
import whois
import scrapy
import unicodedata
import urllib.request
from io import StringIO
from datetime import datetime
from html.parser import HTMLParser
from dateutil import relativedelta

base_url = 'https://www.fma.govt.nz'

"""
    Class to strip html tags
    Returns formatted text
"""
class MLStripper(HTMLParser):

    def __init__(self):
        super().__init__()
        self.reset()
        self.strict = False
        self.convert_charrefs= True
        self.text = StringIO()
    def handle_data(self, d):
        self.text.write(d)
    def get_data(self):
        return self.text.getvalue()

def strip_tags(html):

    s = MLStripper()
    s.feed(html)
    return s.get_data()

"""
    Get article links
    Returns list of url of article link
"""
def get_links(curl_result):

    regex = re.compile(r'(\<a href="/library/warnings-and-alerts/.*?/">)')
    results = re.findall(regex,curl_result)
    formatted_links = [link.split('a href="')[1].split('/">')[0] for link in results]

    return [base_url + link for link in formatted_links]

"""
    Get domain status
    Returns string of domain status
"""
def domain_status(domain):
    status_result = ""
    
    try:
        if(domain == 'No website provided.'):
            return 'No website provided.'

        if 'http' not in domain:
            domain = 'https://' + domain

        status = urllib.request.urlopen(domain).getcode()
        
        if status == 200:
            print('Alive as of: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
            status_result = 'Alive as of: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
        elif status == 404:
            print('Does not exist as of: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
            status_result = 'Does not exist as of: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
        elif status == 503 or status == 500:
            print('Unavailable')
            status_result = 'Unavailable as of: ' + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
        else:
            print('Unknown status:' + str(status) + " as of: " + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S")))
            status_result = 'Unknown status:' + str(status) + " as of: " + str(datetime.now().strftime("%Y-%m-%d %H:%M:%S"))
    except Exception as ex: 
        return domain + ' - Unable to get status of URL: ' + str(ex)
    return domain + " - " + status_result

"""
    Get whois information
    Returns string of whois information otherwise respective error message
"""
def whois_info(domain,date_data):
    date_diff = "Unable to calculate difference."
    if(domain.endswith("/")):
        domain = domain[:-1]
        
    if(domain == 'No website provided.'):
       
        return 'No website provided.',date_diff 
    elif(domain == ""):
        return ""

    try:  
       
        wresults = whois.query(domain)
        if wresults is not None:

            date_diff = calc_creation_diff(str(wresults.creation_date).split(" ")[0],date_data)  
            return "Name: " + wresults.name+"\n" + "Emails: " + ' '.join(wresults.emails)+ '\n' + "Creation Date: " + str(wresults.creation_date) +'\n' + "Last Updated: " + str(wresults.last_updated) +'\n' + "Expiration Date: " +  str(wresults.expiration_date) +'\n' + "Name Servers: " +' '.join(wresults.name_servers), domain + " - " + date_diff
        else:
            return "Unable to acquire whois data for "+domain, date_diff
    except Exception as ex:
        print("Whois - Error: " + str(ex))
        return "Unable to acquire whois data for " + domain, date_diff

"""
    Scrapy spider
"""
class FMASpider(scrapy.Spider):

    name = 'fma-spider'
    start_urls = ['https://www.fma.govt.nz/library/warnings-and-alerts/']

    def parse(self, response):

        # Single request
        # yield scrapy.Request('https://www.fma.govt.nz/library/warnings-and-alerts/principle-forsakrings-holdings/',callback=self.parse_page_data)
      
        # Multiple requests
        next_page = response.css('.next::attr(href)').extract()
        if next_page is not None:
            next_page_link=base_url+''.join(next_page)
            url_list = os.popen("curl "+next_page_link).read()
        
            for url in get_links(url_list):
                yield scrapy.Request(url,callback=self.parse_page_data)
            yield scrapy.Request(url=next_page_link, callback= self.parse)
    """
        Get article data
    """
    def parse_page_data(self, response):
        # Get text of requested elements on webpage
        date_selector = '.published__text::text'
        title_selector = '.registry-item-page__heading-wrap--title-item::text'
        email_selector = '.content-element__content a'
        content_selector = '.content-element__content p'
        content_list_selector = '.content-element__content li'
        normalised_raw_data = unicodedata.normalize("NFKD",str(''.join(response.css(content_selector).extract())))
        # Format element data into respective string
        date_data = ''.join(response.css(date_selector).extract())
        title_data = ''.join(response.css(title_selector).extract())
        content_data = format_content(normalised_raw_data,response.css(content_list_selector).extract()).encode("ascii","ignore").decode().lstrip()
        entity_name_data = format_entity_name(normalised_raw_data)
        website_data = format_website(normalised_raw_data).lstrip()
        address_data = format_address(normalised_raw_data)
        email_data = decodeEmails(response.css(email_selector).extract())
        phone_data = format_phone(response.css(content_selector).extract()) 
        # Acquire domain information
        split_domains = website_data.splitlines()
        whois_list = []
        status_list = []
        date_diff_list = []
        if(len(split_domains) >= 1):
            for domain in split_domains[:-1]:
                whois_results, date_diff = whois_info(domain.strip(),date_data)
                whois_list.append(whois_results+"\n")
                date_diff_list.append(date_diff+"\n")
                status_list.append(domain_status(domain.strip())+"\n")
            whois_results_final, date_diff_final = whois_info(split_domains[-1].strip(),date_data)
            whois_list.append(whois_results_final)
            date_diff_list.append(date_diff_final)
            status_list.append(domain_status(split_domains[-1].strip()))
        else:
            whois_list.append("No website provided.")
        whois_data = ''.join(whois_list)
        status_data = ''.join(status_list)
        date_diff_date = ''.join(date_diff_list)


        csv_row_data = [date_data,"\""+title_data+"\"","\""+content_data+"\"",entity_name_data,website_data,address_data,email_data,phone_data,status_data,whois_data,date_diff_date]
        write_csv_data(csv_row_data)

        yield {}

"""
    Calcualte different of domain creation date and article publishing date
"""
def calc_creation_diff(domain_creation_date,article_date):
    if (domain_creation_date is None or article_date is None):
        return "Unable to calculate difference."
    formatted_article_date = datetime.strptime(article_date,"%d %B %Y")
    formatted_domain_creation_date = datetime.strptime(domain_creation_date,"%Y-%m-%d")
    date_diff = relativedelta.relativedelta(formatted_article_date,formatted_domain_creation_date)
    return str(date_diff.months) + " Months " + str(date_diff.days) + " Days" 

def format_address(content):
    
    regex = re.compile(r'<strong>addresse?s?:</strong>',re.IGNORECASE)
    raw_content = re.split(regex,content)
    if(len(raw_content) > 1):
        website_str = raw_content[1].split('<br>')[0]
        formatted_website_str = website_str.replace(':','')
    else:
        return "No address provided."
    
    return strip_tags(formatted_website_str.split('</p>')[0]).lstrip()

def format_entity_name(content):
    
    regex =  re.compile(r'entity name:',re.IGNORECASE)
    secondary_regex = re.compile(r'<strong>entity title', re.IGNORECASE)
    entity_name = re.split(regex,content)
    secondary_name = re.split(secondary_regex,content)

    if(len(entity_name) > 1):
        
        formatted_entity_name = entity_name[1].split('<br>')[0].replace(': ','')
    elif(len(secondary_name) > 1):
        formatted_entity_name = strip_tags(secondary_name[1].split('<br>')[0]).lstrip().replace(': ','')
    else:
        return "No entity name provided."
    final_entity_name = strip_tags(formatted_entity_name.split('<strong>')[0].strip().replace('; ','\n')).lstrip()

    return final_entity_name

"""
    Process website element
"""
def format_website(content):

    regex = re.compile(r'<strong>websites?:.?</strong>', re.IGNORECASE)
    raw_content = re.split(regex,content)
    if(len(raw_content) > 1):
        end_website_tag =  re.compile(r'<strong>')
        if(len(re.split(end_website_tag,raw_content[1])) <= 1):
            end_website_tag =  re.compile(r'</p>')
            if('href' in raw_content[1]):
                end_website_tag =  re.compile(r'<b>')
        website_str = re.split(end_website_tag,raw_content[1])[0]
        formatted_website_str = website_str.replace(':','').replace('; ','\n').replace(' and ','\n').replace(' AND ', '\n').replace(', ','\n').replace(' and/or ', '\n')
    else:
        return "No website provided."
    
    if('for warning' in strip_tags(formatted_website_str.lower())):
        split_header_regex = re.compile(r'reasons? for warning', re.IGNORECASE)
        removed_header_website = re.split(split_header_regex,strip_tags(formatted_website_str))
        return removed_header_website[0].lstrip()
    regex_white_space = re.compile(r"^\s+",re.MULTILINE)
    return strip_tags(regex_white_space.sub('',formatted_website_str))

def format_content(content,content_list):

    primary_regex =  re.compile(r'entity name',re.IGNORECASE)
    secondary_regex = re.compile(r'reasons? for warning', re.IGNORECASE)
    raw_content =  re.split(primary_regex,content)
    formatted_content_str = strip_tags(raw_content[0])
    formatted_content_list = ''.join(content_list).replace('<li>','\n')
    if(len(raw_content) == 1 or formatted_content_str.strip() == ""):
       
        formatted_content_str = re.split(secondary_regex,content)
        if(len(formatted_content_str) == 1):
            regex_format_line = re.compile(r'(<strong>|<b>)')
            return strip_tags(regex_format_line.sub('\n',''.join(formatted_content_str)))
        if(len(re.split(r'<strong>', formatted_content_str[1])) > 4):
            return strip_tags(formatted_content_str[1].split('<strong>')[0].replace(':','')+formatted_content_list).lstrip()
            
        return strip_tags(formatted_content_str[1].replace(':','')+formatted_content_list).lstrip()
    
    if(len(re.split(secondary_regex,content)) > 1):
       
        cont_str = re.split(secondary_regex,content)[1].split('</strong>')[1]
        return strip_tags(formatted_content_str + "\n" + cont_str.split('</strong>')[0] + formatted_content_list)
    
    return formatted_content_str + strip_tags(formatted_content_list)

"""
    Decode email address string on webpage
"""
def decodeEmails(emails):

    de = ""
    for i in range(0,len(emails)):
        if(len(emails[i].split('data-cfemail=')) > 1):
            de += '\n'
            e = emails[i].split('data-cfemail=')[1].split('"')[1].strip()
            k = int(e[:2], 16)

            for i in range(2, len(e)-1, 2):
                de += chr(int(e[i:i+2], 16)^k) 
    de = ''.join(de)

    if (len(emails) == 0 or de == ""):
        return "No emails provided."
    
    return de

def format_phone(content):
   
    formatted_number = ""
    raw_content = str(content)
    regex = re.compile(r'(\bPhone\b.*</p>)')
    extracted_phone = re.search(regex,raw_content)
    if(extracted_phone is None):
        return 'No phone number provided.' 
    else:
        num_only_regex = re.compile(r'[^0-9+\s]')
        formatted_number = re.sub(num_only_regex, "",extracted_phone[0].split(':')[1].split('</p>')[0].replace('; ','\n').replace('\\xa0', ''))

    return strip_tags(formatted_number).lstrip()

def write_csv_data(data):

    csv_file_name = 'fma_crawled_data.csv'
    file_exist = os.path.isfile(csv_file_name)
    with open(csv_file_name, 'a') as csvfile:

        headers = ['Date Published','Title','Content','Entity Name','Domains','Address','Email','Phone','Website Status', 'Whois Information','Report to Creation Timeframe']
        writer = csv.writer(csvfile, delimiter=',')

        if (not file_exist):
            writer.writerow(headers)

        writer.writerow(data)