using Compat
using Compose
using Gadfly
using GitHub
using HDF5, JLD
using Interact
using MetadataTools
using JSON
using ProgressMeter
using Requests
using Shapefile
using URIParser

auth_token="" #to access Github API
my_auth = authenticate(auth_token)

try if isfile("worldofjulia.jld")
    A = JLD.load("worldofjulia.jld")
    locations = A["locations"]
    juliastargazers = A["juliastargazers"]
    juliawatchers = A["juliawatchers"]
    juliacontributors = A["juliacontributors"]
    allcontributors = A["allcontributors"]
end end

authors=contributors("JuliaLang/julia", auth=my_auth)[1]
println("$(length(authors)) contributors found for JuliaLang/julia")

dl(url::Nullable{URI}, filename, tries = 3) = 
    if isnull(url)
        throw(ArgumentError("Cannot dl($url)"))
    else
        dl(get(url), filename, tries)
    end

dl(url, filename, tries=3) = dl(URI(url), filename, tries)

#Download data from a given URL to a file.
function dl(url::URI, filename, tries = 3)
    isfile(filename) && return #Don't overwrite existing files
    r = nothing
    for i=1:tries
        try
            r = get(url)
            
            r.status == 200 && break

            if contains(r.headers["Content-Type"], "text/html")
                display("text/html", r.data)
            end
            r.status == 302 && break #Redirection
        catch e
            warn(e)
        end
        sleep(3)
    end
    if r!=nothing && r.status == 200
        open(filename, "w") do f
            write(f, r.data)
        end
    else
        warn("Could not download $url\nStatus: $(r.status)")
    end
end

#Download everyone's avatars
function getavatars(authors; verbose::Bool=false)
    const mogrify = `/usr/local/bin/mogrify`
    
    avatarfiles = Dict()
    @showprogress for author in authors
        login = get(author["contributor"].login)
        avatarfilename = string(login, ".png")
        if !isfile(avatarfilename)
            url = author["contributor"].avatar_url
            verbose && info("Downloading avatar for $login")
            dl(url, avatarfilename)
            run(`$mogrify -resize 64x64 $avatarfilename`)
        else
            verbose && info("Avatar for $login already downloaded")
        end
        avatarfiles[login] = avatarfilename
    end
    avatarfiles
end
avatarfiles = getavatars(authors)

function makemontage(filename, authors, avatarfiles; layout=nothing)
    cmd = `montage`
    layout==nothing || (cmd = `$cmd -tile $(layout[1])x$(layout[2])`)
    σ = sortperm([author["contributions"] for author in authors], rev=true)
    for idx in σ
        author = authors[idx]
        login, contribs = get(author["contributor"].login), author["contributions"]
        if !haskey(avatarfiles, login)
            warn("Skipping user $login: no associated entry in avatarfiles")
            continue
        end

        avatarfile = avatarfiles[login]
        if !isfile(avatarfile)
            warn("Skipping user $login: no file $avatarfile")
            continue
        end
        cmd = `$cmd -label "$login\n($contribs)" $avatarfile`
    end
    cmd = `$cmd -geometry 64x64+16+16 -font Helvetica $filename`
end

#Compute montage layout
aspectratio = φ #Golden ratio
ntiles=√(length(authors)/aspectratio)
ntilesx, ntilesy = ceil(Int, aspectratio*ntiles), ceil(Int, ntiles)

cmd = makemontage("montage_juliaonly.jpg", authors, avatarfiles, layout = (ntilesx, ntilesy));

#@time run(cmd)

#Scan package metadata and get URLs for all registered packages
Packages = Any[]
@showprogress for pkg in Pkg.available()
    url = URIParser.parse_url(get_pkg(pkg).url)
    if url.host == "github.com"
        _, owner, repo = try
            split(url.path, '/')
        catch exc
            println(STDERR, "Error parsing $(url.path) from $url")
            rethrow(exc)
        end
        repo = split(repo, ".git")[1]
        push!(Packages, (owner, repo))
    else
        warn("Skipping non-GitHub repo $url")
    end
end
println("$(length(Packages)) packages found.")

#Update authors with package contributors
#Collates all the contribution counts also
function addpkgcounts!(authors, Packages)
    @showprogress for (owner, repo) in Packages
        thispkg_contributors = try
            contributors(owner*"/"*repo, auth=my_auth)[1]
        catch
            warn("Skipping $owner/$repo")
            continue
        end

        isnew = true
        #Merge global statistics
        for entry in thispkg_contributors
            contributor = entry["contributor"]
            for (i, authordata) in enumerate(authors)
                if get(authordata["contributor"].login) == get(contributor.login)
                    isnew = false
                    authors[i]["contributions"] += entry["contributions"]
                    break
                end
            end
            isnew && push!(authors, entry)
        end
    end
end
    
juliacontributors = copy(authors)
addpkgcounts!(authors, Packages)

avatarfiles = getavatars(authors)

#Compute montage layout
aspectratio = φ
ntiles=√(length(authors)/aspectratio)
ntilesx, ntilesy = ceil(Int, aspectratio*ntiles), ceil(Int, ntiles)

cmd = makemontage("montage_julia.jpg", authors, avatarfiles, layout = (ntilesx, ntilesy));

@time run(cmd)

spacing=10
rawcounts = Int[author["contributions"] for author in authors]
uppercount=spacing*ceil(Int, maximum(rawcounts)/spacing)
grid = logspace(0, log10(uppercount), 22)
_, counts = hist(rawcounts, grid)

plot(Geom.bar, Guide.xlabel("Commits"), Guide.ylabel("Number of contributors"),
    y=counts, x=midpoints(grid))

#Convert Shapefile rectangle to Compose rectangle
Compose.rectangle{T<:Real}(R::Shapefile.Rect{T}) = rectangle(R.left,R.top,R.right-R.left,R.bottom-R.top)

#Compose polygons cannot be disjoint but Shapefile.Polygons can
#Need to convert Shapefile.Polygon to list of Compose polygons
function Base.convert(::Type{Vector{Compose.Form{Compose.PolygonPrimitive}}},
        shape::Shapefile.Polygon)
    points = Any[]
    polygons=Any[]
    currentpart=2
    for (i,p) in enumerate(shape.points)
        push!(points, p)
        if i==length(shape.points) || (currentpart≤length(shape.parts) && i==shape.parts[currentpart])
            push!(polygons, polygon([(p.x,p.y) for p in points]))
            currentpart += 1
            points = Any[]
        end
    end
    polygons
end

Polygons(shape::Shapefile.Polygon) = convert(Vector{Compose.Form{Compose.PolygonPrimitive}}, shape)

#Technically correct only for S=Shapefile.ESRIShape
Polygons{S<:Shapefile.ESRIShape}(shapes::Vector{S}) = [[convert(Vector{Compose.Form{Compose.PolygonPrimitive}},
    shape) for shape in shapes]...]

#Load some data about the world's countries
worldshapefile="ne_110m_admin_0_countries.shp"
dl("https://raw.githubusercontent.com/nvkelso/natural-earth-vector/master/110m_cultural/ne_110m_admin_0_countries.shp", worldshapefile)
worldshape = open(worldshapefile) do f
      read(f, Handle)
end
world=compose(context(), fill(nothing), stroke("black"), Polygons(worldshape.shapes)...)
draw(SVG(8inch, 4inch), compose(context(units=UnitBox(-180, 90, 360, -180)), world))

getlatlon(user::AbstractString, my_auth=my_auth) = getlatlon(owner(user, auth=my_auth))

function getlatlon(user::GitHub.Owner, my_auth=my_auth)
    username = get(user.login)
    location = if !isnull(user.location)
        get(user.location) #Read from GitHub.Owner
    else
        try #Work around https://github.com/JuliaWeb/GitHub.jl/issues/51
            #Get user location from GitHub
            user2 = owner(username, auth=my_auth)
            if !isnull(user2.location)
                get(user2.location)
            else
                "" #No location data
            end
        catch
            ""
        end
    end
    strip(location) == "" && return nothing  #Location was missing

    #If user-reported location is already coordinates, then return them
    try
        #test if location is a string of all digits, punctuation or whitespace
        if all(map(x->isdigit(x)|ispunct(x)|isspace(x), collect(location))
            coords = eval(parse(location))
            if typeof(coords) == Tuple{Float64,Float64}
                return (coords[1], coords[2], location)
            end
        end
    end

    #Some simple hacks to normalize locations for Nominatim
    location = replace(location, "U.S.A.", "USA")
    
    #Take location and look up on geocoding service
    try
        responseosm = get(URI("http://nominatim.openstreetmap.org/search"),
        query=@compat Dict("format"=>"json", "q"=>location))
        
        responseosmstr = bytestring(responseosm.data)
        
        if responseosm.status!=200 && contains(responseosm.headers["Content-Type"], "text/html")
            display("text/html", responseosmstr)
        end
        meosm = JSON.parse(responseosmstr)
        if length(meosm)<1
            warn("OpenStreetMaps did not know the location of user $username with reported location \"$location\"")
            return (Inf, Inf, location) #Geocoder doesn't know where this is
        end
        #Return the first hit
        return (float(meosm[1]["lat"]), float(meosm[1]["lon"]), location)
    catch e
        warn("Ignoring bad response from URL: http://nominatim.openstreetmap.org/search?format=json&q=$location")
        println("Error caught: ", e)
        if isdefined(:responseosm)
            if contains(responseosm.headers["Content-Type"], "text/html")
                display("text/html", responseosmstr)
            else
                println(responseosmstr)
            end
        end
        return nothing
    end
end

getlatlon("jiahao", my_auth)

function adduserlocations!(locations::Dict, contributors::Vector{GitHub.Owner})
    @showprogress for user in contributors
        adduserlocation!(locations, user)
    end
end

function adduserlocations!(locations::Dict, contributors::Vector{Any})
    @showprogress for user in contributors
        adduserlocation!(locations, user["contributor"])
    end
end

function adduserlocation!(locations::Dict, user::GitHub.Owner)
    username = get(user.login)
    haskey(locations, username) && return #Don't look up existing data again
    location = getlatlon(user, my_auth)
    location==nothing && return
    locations[username] = location
end

isdefined(:locations) || (locations=Dict())
adduserlocations!(locations, authors)

#Render developers
function placeauthors(authors, locations)
    xs = Float64[]; ys=Float64[]; rs=Any[]
    @showprogress for user in authors
        username, n = if isa(user, Dict)
            get(user["contributor"].login), user["contributions"]
        elseif isa(user, GitHub.Owner)
            get(user.login), 1
        end

        haskey(locations, username) || continue
        push!(xs, locations[username][2])
        push!(ys, locations[username][1])
        push!(rs, (1+√log(n))*0.3mm)
    end
    circle(xs, ys, rs)
end

function drawmap(left::Real=-180, right::Real=180, up::Real=90, down::Real=-90,
    composeobjs...; target = SVG(4inch*(right-left)/(up-down),4inch))
    draw(target,
    compose(context(units=UnitBox(left,up,right-left,down-up)), world, composeobjs...))

    #Print users in the box
    #for (user, loc) in locations
    #    if down<loc[1]<up && left<loc[2]<right
    #        println(user, ":", loc)
    #    end
    #end
end

devs=compose(context(), fill("#d66661"), stroke("#c93d39"), linewidth(0.3mm),
        placeauthors(authors, locations))

drawmap(-180, 180, 90, -90, devs)

@manipulate for limit_east=-180:180, limit_west=limit_east.value:180, limit_north=-90:90, limit_south=-90:limit_north.value
    left = limit_east
    right = limit_west
    up = limit_north
    down = limit_south
    box = UnitBox(left, up, right-left, down-up)
    compose(context(units=box), world, devs)
end

adduserlocations!(locations, juliastargazers)

# Get stargazers
juliastargazers = stargazers("JuliaLang/julia", auth=my_auth)[1]
adduserlocations!(locations, juliastargazers)

# Get watchers
juliawatchers = watchers("JuliaLang/julia", auth=my_auth)[1]
adduserlocations!(locations, juliawatchers)

#Render stargazers and watches
stars=compose(context(), fill(nothing), stroke("#945bb0"),
    placeauthors(juliastargazers, locations))

watches=compose(context(), fill("#6bab5b"),
    placeauthors(juliawatchers, locations));

drawmap(-180, 180, 90, -90, devs, stars, watches)

drawmap(-180, 180, 90, -90, devs, stars, watches, target=PNG("worldofjulia.png", 8inch, 4inch))

@manipulate for limit_east=-180:180, limit_west=limit_east.value:180, limit_north=-90:90, limit_south=-90:limit_north.value
    left = limit_east
    right = limit_west
    up = limit_north
    down = limit_south
    box = UnitBox(left, up, right-left, down-up)
    compose(context(units=box), world, devs, stars, watches)
end

JLD.save("worldofjulia.jld",
    #Currently can't be saved - https://github.com/JuliaLang/JLD.jl/issues/61
    #"allcontributors", authors,
    #"juliacontributors", juliacontributors,
    #"juliastargazers", juliastargazers, 
    #"juliawatchers", juliawatchers,
    "locations", locations
)