#!/usr/bin/env python # coding: utf-8 # In[1]: import sys print(sys.executable) print(sys.version) print(sys.version_info) # In[2]: import pysal as ps #import fiona import numpy as np import pandas as pd import geopandas as gpd # In[4]: get_ipython().system('pwd') # In[5]: ps.examples.available() # In[6]: ps.examples.explain('nyc_bikes') # In[7]: ps.examples.explain('tokyo') # In[8]: ps.examples.explain('us_income') # In[10]: print(ps.examples) # print(" "*90) # print(ps.examples.get_path) # In[11]: get_ipython().system('cd /home/dhankar/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/examples') # In[12]: get_ipython().system('pwd') # In[ ]: """ (gisenv) dhankar@dhankar-VPCEB44EN:~/Desktop/Geo_1$ (gisenv) dhankar@dhankar-VPCEB44EN:~/Desktop/Geo_1$ pwd /home/dhankar/Desktop/Geo_1 (gisenv) dhankar@dhankar-VPCEB44EN:~/Desktop/Geo_1$ (gisenv) dhankar@dhankar-VPCEB44EN:~/Desktop/Geo_1$ cd /home/dhankar/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/examples (gisenv) dhankar@dhankar-VPCEB44EN:~/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/examples$ ls 10740 burkitt columbus georgia londonhp newHaven Polygon_Holes sids2 street_net_pts tokyo arcgis calemp desmith __init__.py mexico nyc_bikes __pycache__ snow_maps taz us_income baltim chicago examples.txt juvenile nat Point README.txt south test_examples.py virginia book clearwater geodanet Line networks Polygon sacramento2 stl tmpIvIYa5.gwt wmat (gisenv) dhankar@dhankar-VPCEB44EN:~/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/examples$ (gisenv) dhankar@dhankar-VPCEB44EN:~/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/examples$ cd us_income (gisenv) dhankar@dhankar-VPCEB44EN:~/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/examples/us_income$ ls README.md spi_download.csv states48.gal us48.dbf us48.shp us48.shx usjoin.csv (gisenv) dhankar@dhankar-VPCEB44EN:~/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/examples/us_income$ (gisenv) dhankar@dhankar-VPCEB44EN:~/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/examples/us_income$ pwd /home/dhankar/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/examples/us_income (gisenv) dhankar@dhankar-VPCEB44EN:~/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/examples/us_income$ (gisenv) dhankar@dhankar-VPCEB44EN:~/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/examples/us_income$ """ # In[3]: csv_path = ps.examples.get_path('usjoin.csv') print("CSV Local File Path___ ",csv_path) # In[4]: f = ps.open(csv_path) # print(" "*10) # print(f.header[0]) ## Column 1 of CVS File # print(" "*10) # print(f.header[5]) # print(" "*10) # print(f.header) # print(" "*10) #print(type(f)) ## # In[5]: year_09 = f.by_col('2009') print(year_09) # In[6]: ## What are .shp -SHAPE Files ? --- https://www.esri.com/library/whitepapers/pdfs/shapefile.pdf ## PySAL reading -SHAPE Files --- ## path_shp_file = "/media/dhankar/Dhankar_1/a1_18/a1____GeoSpatialANAL/GIS_DATA/ShapeFiles/tl_2010_02_place10.shp" print("path_shp_file____",path_shp_file) # shp_f = ps.open(path_shp_file) # print(type(shp_f)) # # # In[7]: # Exploring SHAPE File print(shp_f.header) # print(" "*10) # print(shp_f.header[0]) # Cant Index --PurePyShpWrapper -- like the - csvWrapper -- File Object above. # print(" "*10) # In[33]: # The Bounding Boxes shp_f.bbox # In[37]: # Source -- http://nbviewer.jupyter.org/github/pysal/notebooks/blob/master/notebooks/PySAL_io.ipynb # ploygons_list =[] # for poly in shp_f: # print("Iterating over polygon: ", poly.id) ## OK Prints ALL POLYGON ID's ploygons_list.append(poly) print(len(ploygons_list)) # In[41]: # Polygon - ONE of SHAPE File poly1 = shp_f[0] print(type(poly1)) # print("poly1.area_____",poly1.area) # print("poly1.centroid_____",poly1.centroid) # print("poly1.holes_____",poly1.holes) # print("poly1.perimeter_____",poly1.perimeter) # # In[43]: # Writing Own SHAPE Files -- Creating a COPY # # Create a NEW DUMMY SHAPE File on local DIR shp_f1 = ps.open('shp_test.shp', 'w') # # Copy the Headers shp_f1.header = shp_f.header # # Write ONE Polygon to New Shape File # shp_f1.write(poly1) shp_f1.close() # print(type(shp_f1)) # In[8]: shp_f1 = ps.open('/home/dhankar/Desktop/Geo_1/shp_test.shp', 'w') print(shp_f1) ## WIP --- Check Below ---- # Polygon - ONE of NEW SHAPE File == shp_f1 poly_1_1 = shp_f1[0] print(type(poly_1_1)) # print("poly1.area_____",poly_1_1.area) # print("poly1.centroid_____",poly_1_1.centroid) # print("poly1.holes_____",poly_1_1.holes) # print("poly1.perimeter_____",poly1.perimeter) # # # DBF Files # ## The dBase - dbf Files Wiki --- https://en.wikipedia.org/wiki/DBase # # # In[9]: dbf_path = ps.examples.get_path('us48.dbf') print(".dbf Local File Path___ ",dbf_path) # In[59]: # explore the .dbf file dbf_1 = ps.open(dbf_path) print(type(dbf_1)) # print(" "*10) # print(dbf_1.header[0]) # print(" "*10) # print(dbf_1.header[1]) # print(" "*10) # print(dbf_1.header[2]) # print(" "*10) # print(dbf_1.header[3]) # print(" "*10) # print(dbf_1.header[4:7]) # print(" "*10) # print(dbf_1.field_spec) # # ### SOURCE -- http://pysal.readthedocs.io/en/latest/library/core/IOHandlers/pyDbfIO.html # # field_spec == A list describing the data types of each field. # It is comprised of a list of tuples, each tuple describing a field. # The format for the tuples is (“Type”,len,precision). # Valid Types are ‘C’ for characters, ‘L’ for bool, ‘D’ for data, ‘N’ or ‘F’ for number. # # # In[58]: # Print values from any ONE COLUMN - with COLUMN LABEL # We get LIST Object state_id = dbf_1.by_col("STATE_ID") print(state_id) # print(" "*40) # print(state_id[0:5]) # print(" "*40) # print(type(state_id)) # In[18]: # Get a NUMPY.nd.array from the dbf file columns ids_np_array = np.array([dbf_1.by_col(ids) for ids in dbf_1.header]) # t_ids_np_array = np.array([dbf_1.by_col(ids) for ids in dbf_1.header]).T ### TRANSPOSED # print(ids_np_array.shape) # print("TRANSPOSED Shape ==",t_ids_np_array.shape) # print(" "*90) # # As seen below - we have "All Columns Values" in a 2D Numpy Array - we dont have COLUMN LABELS # Not intelligent... print(ids_np_array) # print(" "*90) # # We TRANSPOSED the Numpy Array to show it in a DATAFRAME ROWS like format # print("TRANSPOSED==",t_ids_np_array) # # In[17]: # Accessing all values within a COLUMN states = dbf_1.by_col("STATE_NAME") print(states) # In[20]: ## Extracting Column Data -- with specific Value Types ## Writing a List Comprehension ## NUMERIC Only Columns num_cols = np.array([dbf_1.by_col(dbf_1.header[i]) for i in range(len(dbf_1.header)) if dbf_1.field_spec[i][0] == 'N']).T print(type(num_cols)) print(" "*20) print("SHAPE of the numpy Array ____",num_cols.shape) ## 2 D Array with SHAPE - (48, 4) print(" "*20) print(num_cols) # In[23]: str_cols = np.array([dbf_1.by_col(dbf_1.header[i]) for i in range(len(dbf_1.header)) if dbf_1.field_spec[i][0] == 'C']).T print(type(str_cols)) print(" "*20) print("SHAPE of the numpy Array ____",str_cols.shape) ## 2 D Array with SHAPE - (48, 4) print(" "*20) print(str_cols) # In[36]: # Creating a DICTIONARY n then a PANADAS DataFrame from the Numpy Arrays # dic_1 = dict([(col, np.array(dbf_1.by_col(col))) for col in dbf_1.header]) df_1 = pd.DataFrame(dic_1) # Print out Top Five and Bottom Five Observations / Rows print(df_1.head(5)) print("_ _"*60) print(" "*60) print(df_1.tail(5)) # In[37]: # to get Original Order of Columns - pass the dbf_1.header , LIST , to the PANDAS DF as a LIST index # print(type(dbf_1.header)) print(" "*60) #df_1 = df_1[dbf_1.header] print(df_1.head(5)) print("_ _"*60) print(" "*60) print(df_1.tail(5)) # In[39]: print(df_1.describe()) print(" "*60) print(df_1.describe().T) # In[50]: us_income_sh_path = ps.examples.get_path('us48.shp') print(".dbf Local File Path___ ",us_income_sh_path) # In[57]: # Visualizing the POLYGONS # get_ipython().run_line_magic('matplotlib', 'inline') import matplotlib import matplotlib.pyplot as plt from pysal.contrib.viz import mapping as maps # fig = plt.figure() # #path_shp_file = "/media/dhankar/Dhankar_1/a1_18/a1____GeoSpatialANAL/GIS_DATA/ShapeFiles/tl_2010_02_place10.shp" # #print("path_shp_file____",path_shp_file) # shp_f = ps.open(us_income_sh_path) # print(type(shp_f)) # base = maps.map_poly_shp(shp_f) base.set_facecolor('blue') base.set_linewidth(0.75) base.set_edgecolor('0.3') ## 0.8 --- gives Light Greyish edges ax = maps.setup_ax([base],[shp_f.bbox, shp_f.bbox, shp_f.bbox]) print(type(ax)) fig.add_axes(ax) plt.show() #maps.plot_poly_lines(path_shp_file) # In[60]: london_hp = ps.examples.get_path('londonhp.shp') ## This FUNC searches in the "examples" DIR for File Name STRING being passed. print(".shp Local File Path___ ",london_hp) # In[62]: # Visualizing the POLYGONS # get_ipython().run_line_magic('matplotlib', 'inline') import matplotlib import matplotlib.pyplot as plt from pysal.contrib.viz import mapping as maps # fig1 = plt.figure() # shp_f1 = ps.open(london_hp) # print(type(shp_f1)) # base = maps.map_poly_shp(shp_f1) base.set_facecolor('blue') base.set_linewidth(0.75) base.set_edgecolor('0.3') ## 0.8 --- gives Light Greyish edges ax1 = maps.setup_ax([base],[shp_f1.bbox, shp_f1.bbox, shp_f1.bbox]) print(type(ax1)) fig1.add_axes(ax1) plt.show() #maps.plot_poly_lines(path_shp_file) # In[63]: natregimes_shp = ps.examples.get_path('natregimes.shp') ## This FUNC searches in the "examples" DIR for File Name STRING being passed. print(".shp Local File Path___ ",natregimes_shp) # In[69]: # Visualizing the POLYGONS # get_ipython().run_line_magic('matplotlib', 'inline') import random as rdm import matplotlib import matplotlib.pyplot as plt from pysal.contrib.viz import mapping as maps # fig2 = plt.figure(figsize=(9, 9), dpi=100) # shp_f2 = ps.open(natregimes_shp) # print(type(shp_f2)) # base = maps.map_poly_shp(shp_f2) base.set_facecolor('pink') base.set_linewidth(0.25) base.set_edgecolor('0.3') ## 0.8 --- gives Light Greyish edges ax2 = maps.setup_ax([base],[shp_f2.bbox, shp_f2.bbox, shp_f2.bbox]) print(type(ax2)) # examples.get_path('natregimes.shp') fig2.add_axes(ax2) plt.show() # In[75]: # Level TWO Visualization with PySAL # SOURCE --- http://darribas.org/gds_scipy16/ipynb_md/02_geovisualization.html # # net_link = ps.examples.get_path('eberly_net.shp') net_link_dbf = ps.examples.get_path('eberly_net.dbf') print(net_link) print(net_link_dbf) # net = ps.open(net_link) values = np.array(ps.open(net_link.replace('.shp', '.dbf')).by_col('TNODE')) # Above line of Code == stores VALUES of COLUMN TNODE - from dbase .dbf File # pts_link = ps.examples.get_path('eberly_net_pts_onnetwork.shp') print(pts_link) pts = ps.open(pts_link) # fig_l2 = plt.figure(figsize=(9,9)) ### CHECK --- BOKEH or MATPLOTLIB Figure ? # netm = maps.map_line_shp(net) netc = maps.base_choropleth_unique(netm, values) # ptsm = maps.map_point_shp(pts) ptsm = maps.base_choropleth_classif(ptsm, values) ptsm.set_alpha(0.5) ptsm.set_linewidth(0.) # ax = maps.setup_ax([netc, ptsm], [net.bbox, net.bbox]) fig.add_axes(ax) show() # In[7]: # Online TEXAS Shape Files """ blockgroup: https://www2.census.gov/geo/tiger/TIGER2010/BG/2010/tl_2010_48_bg10.zip state: https://www2.census.gov/geo/tiger/TIGER2010/STATE/2010/tl_2010_48_state10.zip county: https://www2.census.gov/geo/tiger/TIGER2010/COUNTY/2010/tl_2010_48_county10.zip tract: https://www2.census.gov/geo/tiger/TIGER2010/TRACT/2010/tl_2010_48_tract10.zip cd: https://www2.census.gov/geo/tiger/TIGER2010/CD/111/tl_2010_48_cd111.zip block: https://www2.census.gov/geo/tiger/TIGER2010/TABBLOCK/2010/tl_2010_48_tabblock10.zip zcta: https://www2.census.gov/geo/tiger/TIGER2010/ZCTA5/2010/tl_2010_48_zcta510.zip """ tex_shp = "/home/dhankar/Desktop/Geo_1/data/texas.shp" # shp_tex = ps.open(tex_shp) # print(type(shp_tex)) # print(" "*10) # print(shp_tex.header) # print(" "*10) # print(shp_tex.header[0]) # Cant Index --PurePyShpWrapper -- like the - csvWrapper -- File Object above. # print(" "*10) # In[8]: # The Bounding Boxes shp_tex.bbox # In[9]: # Source -- http://nbviewer.jupyter.org/github/pysal/notebooks/blob/master/notebooks/PySAL_io.ipynb # We got only the EXTERNAL STATE Boundary Polygon for TEXAS ? ploygons_list =[] # for poly in shp_tex: # print("Iterating over polygon: ", poly.id) ## OK Prints ALL POLYGON ID's ploygons_list.append(poly) print(len(ploygons_list)) # In[10]: # Polygon - ONE of SHAPE File poly1 = shp_tex[0] print(type(poly1)) # print("poly1.area_____",poly1.area) # print("poly1.centroid_____",poly1.centroid) # print("poly1.holes_____",poly1.holes) # print("poly1.perimeter_____",poly1.perimeter) # # In[21]: get_ipython().run_line_magic('matplotlib', 'inline') import numpy as np import pysal as ps import random as rdm from pysal.contrib.viz import mapping as maps tex_shp_path = "/home/dhankar/Desktop/Geo_1/data/texas.shp" tex_cnts_shp_path = "/home/dhankar/Desktop/Geo_1/data/texas_counties/tl_2010_48_county10.shp" tex_dbf_path = '/home/dhankar/Desktop/Geo_1/data/texas.dbf' tex_cnts_dbf_path = '/home/dhankar/Desktop/Geo_1/data/texas_counties/tl_2010_48_county10.dbf' tex_dbf = ps.open(tex_dbf_path) tex_cnts_dbf = ps.open(tex_cnts_dbf_path) # # explore the .dbf file print(type(tex_dbf)) # print(" "*10) # print(tex_dbf.header) print(tex_cnts_dbf.header) # print(" "*60) # print(tex_dbf.header[0]) print(tex_cnts_dbf.header[0]) # print(" "*60) # # Get a NUMPY.nd.array from the dbf file columns # tex_np_array = np.array([tex_dbf.by_col(ids) for ids in tex_dbf.header]) ## Texas State t_tex_np_array = np.array([tex_dbf.by_col(ids) for ids in tex_dbf.header]).T ## TRANSPOSED - Texas State # # tex_cnts_np_array = np.array([tex_cnts_dbf.by_col(ids) for ids in tex_cnts_dbf.header]) ## Texas State Counties t_tex_cnts_np_array = np.array([tex_cnts_dbf.by_col(ids) for ids in tex_cnts_dbf.header]).T ## TRANSPOSED - Texas State Counties # # print(tex_np_array.shape) print("TRANSPOSED Shape ==",t_tex_np_array.shape) print(tex_cnts_np_array.shape) print("TRANSPOSED Shape ==",t_tex_cnts_np_array.shape) # # print(" "*60) # # As seen below - we have "All Columns Values" in a 2D Numpy Array - we dont have COLUMN LABELS # Not intelligent... print(ids_np_array) # print(" "*90) # # We TRANSPOSED the Numpy Array to show it in a DATAFRAME ROWS like format # print("TRANSPOSED==",t_ids_np_array) values = np.array(ps.open('/home/dhankar/Desktop/Geo_1/data/texas.dbf').by_col('NAME10')) types = ['classless', 'unique_values', 'quantiles', 'equal_interval', 'fisher_jenks'] for typ in types: # maps.plot_choropleth(tex_shp,values,typ) # # In[ ]: # In[ ]: # In[ ]: # Online TEXAS Shape Files """ blockgroup: https://www2.census.gov/geo/tiger/TIGER2010/BG/2010/tl_2010_48_bg10.zip state: https://www2.census.gov/geo/tiger/TIGER2010/STATE/2010/tl_2010_48_state10.zip county: https://www2.census.gov/geo/tiger/TIGER2010/COUNTY/2010/tl_2010_48_county10.zip tract: https://www2.census.gov/geo/tiger/TIGER2010/TRACT/2010/tl_2010_48_tract10.zip cd: https://www2.census.gov/geo/tiger/TIGER2010/CD/111/tl_2010_48_cd111.zip block: https://www2.census.gov/geo/tiger/TIGER2010/TABBLOCK/2010/tl_2010_48_tabblock10.zip zcta: https://www2.census.gov/geo/tiger/TIGER2010/ZCTA5/2010/tl_2010_48_zcta510.zip """ tex_shp = "/home/dhankar/Desktop/Geo_1/data/texas.shp" shp_tex = ps.open(tex_shp) # print(type(shp_tex)) """ # Without Specifically asking for a .shx file --- the package is searching for a .shx file when .shp is passed... --------------------------------------------------------------------------- FileNotFoundError Traceback (most recent call last) in () 14 tex_shp = "/home/dhankar/Desktop/Geo_1/data/texas.shp" 15 ---> 16 shp_tex = ps.open(tex_shp) 17 # 18 print(type(shp_tex)) ~/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/core/IOHandlers/pyShpIO.py in __init__(self, *args, **kwargs) 66 self.dataObj = None 67 if self.mode == 'r' or self.mode == 'rb': ---> 68 self.__open() 69 elif self.mode == 'w' or self.mode == 'wb': 70 self.__create() ~/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/core/IOHandlers/pyShpIO.py in __open(self) 77 78 def __open(self): ---> 79 self.dataObj = shp_file(self.dataPath) 80 self.header = self.dataObj.header 81 self.bbox = self.dataObj.bbox ~/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/core/util/shapefile.py in __init__(self, fileName, mode, shape_type) 219 220 if mode == 'r': --> 221 self._open_shp_file() 222 elif mode == 'w': 223 if shape_type not in self.SHAPE_TYPES: ~/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/core/util/shapefile.py in _open_shp_file(self) 249 fileName = self.fileName 250 self.fileObj = open(fileName + '.shp', 'rb') --> 251 self._shx = shx_file(fileName) 252 self.header = _unpackDict(UHEADERSTRUCT, self.fileObj) 253 self.shape = TYPE_DISPATCH[self.header['Shape Type']] ~/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/core/util/shapefile.py in __init__(self, fileName, mode) 440 441 if mode == 'r': --> 442 self._open_shx_file() 443 elif mode == 'w': 444 self._create_shx_file() ~/anaconda2/envs/gisenv/lib/python3.4/site-packages/pysal/core/util/shapefile.py in _open_shx_file(self) 462 463 self.__isreadable() --> 464 self.fileObj = open(self.fileName + '.shx', 'rb') 465 self._header = _unpackDict(UHEADERSTRUCT, self.fileObj) 466 self.numRecords = numRecords = (self._header['File Length'] - 50) // 4 FileNotFoundError: [Errno 2] No such file or directory: '/home/dhankar/Desktop/Geo_1/data/texas.shx' """ # In[ ]: # In[ ]: # In[ ]: # In[79]: ### GET DATA Points out of this FILE only for TEXAS ... # Visualizing the POLYGONS of NAT.shp # get_ipython().run_line_magic('matplotlib', 'inline') import random as rdm import matplotlib import matplotlib.pyplot as plt from pysal.contrib.viz import mapping as maps # fig3 = plt.figure(figsize=(9, 9), dpi=100) # shp_f3 = ps.open(nat_shp) # print(type(shp_f3)) # base = maps.map_poly_shp(shp_f3) base.set_facecolor('white') base.set_linewidth(0.25) base.set_edgecolor('0.2') ## 0.8 --- gives Light Greyish edges ax3 = maps.setup_ax([base],[shp_f3.bbox, shp_f3.bbox, shp_f3.bbox]) print(type(ax3)) fig3.add_axes(ax3) plt.show() # In[ ]: # In[ ]: # In[ ]: # In[ ]: # In[ ]: # In[ ]: # In[43]: #SOURCE -- https://github.com/pysal/notebooks/blob/master/notebooks/PySAL_io.ipynb # Hold for now """ f = figure() ax = f.add_subplot(111) poly_collection = viz.map_poly_shp(ps.open(path_shp_file)) poly_collection.set_facecolor('none') ax.add_collection(poly_collection) ax.autoscale_view() ax.set_frame_on(False) ax.axes.get_yaxis().set_visible(False) ax.axes.get_xaxis().set_visible(False) show() """ # In[ ]: # In[ ]: # In[ ]: ## What are .shp -SHAPE Files ? --- https://www.esri.com/library/whitepapers/pdfs/shapefile.pdf ## PySAL reading -SHAPE Files --- ## path_shp_file = "/media/dhankar/Dhankar_1/a1_18/a1____GeoSpatialANAL/GIS_DATA/ShapeFiles/tl_2010_02_place10.shp" print("path_shp_file____",path_shp_file) # shp_f = ps.open(path_shp_file) # print(type(shp_f)) # # # In[ ]: # In[ ]: # In[ ]: # In[ ]: # In[ ]: # In[ ]: # In[ ]: # ## MARKOV based methods - significance in GEOMATICS # # # # In[ ]: ## MARKOV based methods - significance in GEOMATICS # https://brilliant.org/wiki/markov-chains/ # https://en.wikipedia.org/wiki/Markov_chain # Cross refer the CRAN -- # https://cran.r-project.org/web/packages/markovchain/vignettes/an_introduction_to_markovchain_package.pdf # In[ ]: # In[ ]: # In[ ]: # In[ ]: # In[ ]: # In[ ]: # In[ ]: # In[ ]: