# Importing libraries
import requests
import bs4
from bs4 import BeautifulSoup
import pandas as pd

# URLs to scrape
# This is a dictionary object with URLs. 
# We will use this dictionary to scrape information for each url at a time.
urls = {
    'north': 'http://citypopulation.de/en/newzealand/northisland',
    'south': 'http://citypopulation.de/en/newzealand/southisland/'
}

bs_output = BeautifulSoup(markup=output.text, features="html.parser")

# Our data is in the <table> tag with id='ts'
#
table_output = bs_output.find(name='table', attrs={'id': 'ts'})

# Extracting table output which is in <tbody> tag
#
table_body = table_output.find_all('tbody')

north_island_output = []
for item in table_body:
    rows = item.find_all('tr') # extracts <tr> elements in <tbody>
    for row in rows:
        td = row.find_all('td') # extracts <td> elements in each row i.e. <tr>
        td_values = [val.text for val in td] # extracts value of each <td>
        north_island_output.append(td_values) # appends values to the list

north_island_output = []
for item in table_body:
    rows = item.find_all('tr') # extracts <tr> elements in <tbody>
    for row in rows:
        td = row.find_all('td') # extracts <td> elements in each row i.e. <tr>
        td_values = [val.text for val in td] # extracts value of each <td>
        north_island_output.append(td_values[:-1]) # appends values to the list, also excludes the last value that is not needed

north_data = pd.DataFrame(
    data=north_island_output,
    columns=table_columns
)

# Importing libraries
import requests
from bs4 import BeautifulSoup
import pandas as pd

# Variables
south_island_output = []
north_island_output = []
table_columns = []

# URLs for North and South islands
urls = {
    'north': 'http://citypopulation.de/en/newzealand/northisland',
    'south': 'http://citypopulation.de/en/newzealand/southisland/'
}

# Function that downloads the data
def download_data():
    """
    Function extracts td values by looping each child element of the parent.
    
    Two empty lists south_island_output and north_island_output are initialised.
    
    A urls dictionary object with north island and south island urls is also initialised.
    
    Pseudo code:
    - for each item in the dictionary
        - connect to the url
            - if success (response code == 200), then loop through the page data
                - for each row_item in body (loop - look for <tr> element):
                    - for each row in the row_item (loop and look for <td> element):
                        - for each <td> element, extract the text value
                            - finally append those text values into output list
    """    
    for url in urls:
        print(url, urls[url])
        
        ## response
        response = requests.get(urls[url])
        
        if response.status_code == 200:
            print('Response code is 200. Success!')
            try:
                ## web scraping
                soup = BeautifulSoup(response.text, "html.parser")
                table = soup.find(name='table', attrs={'id': 'ts'})
                table_columns.append([x.get_text() for x in table.find_all('th')][:-1])
                body = table.find_all('tbody')
                for item in body:
                    rows = item.find_all('tr')    
                    for row in rows:
                        td = row.find_all('td')
                        td_values = [val.text for val in td]
                        if url == 'north':
                            north_island_output.append(td_values[:-1]) # excluding last column that has an arrow as a value
                        else:
                            south_island_output.append(td_values[:-1]) # excluding last column that has an arrow as a value
            except Exception as ex:
                print(str(ex))
        else:
            print('Oops! {0}'.format(response.status_code))

# North island dataframe
north_island_data = pd.DataFrame(data=north_island_output, columns=table_columns[0])

# South island dataframe
south_island_data = pd.DataFrame(data=south_island_output, columns=table_columns[0])

north_island_data = north_island_data[['Name', 'Status','Region', 'PopulationEstimate2018-06-30']]
south_island_data = south_island_data[['Name', 'Status','Region', 'PopulationEstimate2018-06-30']]

north_island_data['PopulationEstimate2018-06-30'] = north_island_data['PopulationEstimate2018-06-30'].str.replace(',', '')

north_island_data['PopulationEstimate2018-06-30'] = pd.to_numeric(north_island_data['PopulationEstimate2018-06-30'])

south_island_data['PopulationEstimate2018-06-30'] = south_island_data['PopulationEstimate2018-06-30'].str.replace(',', '')

south_island_data['PopulationEstimate2018-06-30'] = pd.to_numeric(south_island_data['PopulationEstimate2018-06-30'])