#load "Paket.fsx" Paket.Dependencies.Install """ frameworks: net45 source https://nuget.org/api/v2 nuget FSharp.Data nuget XPlot.GoogleCharts """ Paket.LoadingScripts.ScriptGeneration.generateScriptsForRootFolder Paket.LoadingScripts.ScriptGeneration.FSharp (Paket.FrameworkIdentifier.DotNetFramework Paket.FrameworkVersion.V4_5) (System.IO.DirectoryInfo __SOURCE_DIRECTORY__) #load "paket-files/include-scripts/net45/include.main.group.fsx" //#load "XPlot.Plotly.Paket.fsx" //#load "XPlot.Plotly.fsx" open System open System.IO open FSharp.Data open FSharp.Data.JsonExtensions open XPlot open XPlot.GoogleCharts open IfSharp.Kernel.App @"" |> Util.Html |> Display type XPlot.GoogleCharts.GoogleChart with member __.GetContentHtml() = let html = __.GetInlineHtml() html .Replace ("google.setOnLoadCallback(drawChart);", "google.load('visualization', '1.0', { packages: ['corechart'], callback: drawChart })") type XPlot.GoogleCharts.Chart with static member Content (chart : GoogleChart) = { ContentType = "text/html"; Data = chart.GetContentHtml() } AddDisplayPrinter (fun (plot: XPlot.GoogleCharts.GoogleChart) -> { ContentType = "text/html"; Data = plot.GetContentHtml() }) type EpisodeInformation = { Id : string; Season : int Episode : int } let episodeInformation = [ { Id = "tt0684181"; Season = 1; Episode = 1 }; { Id = "tt0684157"; Season = 1; Episode = 2 }; { Id = "tt0684145"; Season = 1; Episode = 3 }; { Id = "tt0684186"; Season = 1; Episode = 4 }; { Id = "tt0684151"; Season = 1; Episode = 5 }; { Id = "tt0684165"; Season = 1; Episode = 6 }; { Id = "tt0684161"; Season = 2; Episode = 1 }; { Id = "tt0684146"; Season = 2; Episode = 2 }; { Id = "tt0684180"; Season = 2; Episode = 3 }; { Id = "tt0684177"; Season = 2; Episode = 4 }; { Id = "tt0684175"; Season = 2; Episode = 5 }; { Id = "tt0684169"; Season = 2; Episode = 6 }; { Id = "tt0684144"; Season = 3; Episode = 1 }; { Id = "tt0767232"; Season = 3; Episode = 2 }; { Id = "tt0684172"; Season = 3; Episode = 3 }; { Id = "tt0684148"; Season = 3; Episode = 4 }; { Id = "tt0684185"; Season = 3; Episode = 5 }; { Id = "tt0684183"; Season = 3; Episode = 6 }; { Id = "tt0684149"; Season = 4; Episode = 1 }; { Id = "tt0684152"; Season = 4; Episode = 2 }; { Id = "tt0684160"; Season = 4; Episode = 3 }; { Id = "tt0684187"; Season = 4; Episode = 4 }; { Id = "tt0684153"; Season = 4; Episode = 5 }; { Id = "tt0684164"; Season = 4; Episode = 6 }; { Id = "tt0684159"; Season = 5; Episode = 1 }; { Id = "tt0684182"; Season = 5; Episode = 2 }; { Id = "tt0684179"; Season = 5; Episode = 3 }; { Id = "tt0684174"; Season = 5; Episode = 4 }; { Id = "tt0756588"; Season = 5; Episode = 5 }; { Id = "tt0684143"; Season = 5; Episode = 6 }; { Id = "tt0684173"; Season = 6; Episode = 1 }; { Id = "tt0684163"; Season = 6; Episode = 2 }; { Id = "tt0684158"; Season = 6; Episode = 3 }; { Id = "tt0684155"; Season = 6; Episode = 4 }; { Id = "tt0684176"; Season = 6; Episode = 5 }; { Id = "tt0756589"; Season = 6; Episode = 6 }; { Id = "tt0684184"; Season = 7; Episode = 1 }; { Id = "tt0684178"; Season = 7; Episode = 2 }; { Id = "tt0684168"; Season = 7; Episode = 3 }; { Id = "tt0684154"; Season = 7; Episode = 4 }; { Id = "tt0756587"; Season = 7; Episode = 5 }; { Id = "tt0684147"; Season = 7; Episode = 6 }; { Id = "tt0684156"; Season = 7; Episode = 7 }; { Id = "tt0684166"; Season = 7; Episode = 8 }; { Id = "tt0684140"; Season = 8; Episode = 1 }; { Id = "tt0684141"; Season = 8; Episode = 2 }; { Id = "tt0684142"; Season = 8; Episode = 3 }; { Id = "tt0684150"; Season = 8; Episode = 4 }; { Id = "tt0684162"; Season = 8; Episode = 5 }; { Id = "tt0684170"; Season = 8; Episode = 6 }; { Id = "tt0684171"; Season = 8; Episode = 7 }; { Id = "tt0684167"; Season = 8; Episode = 8 }; { Id = "tt1365540"; Season = 9; Episode = 1 }; { Id = "tt1371606"; Season = 9; Episode = 2 }; { Id = "tt1400975"; Season = 9; Episode = 3 }; { Id = "tt1997038"; Season = 10; Episode = 1 }; { Id = "tt1999714"; Season = 10; Episode = 2 }; { Id = "tt1999715"; Season = 10; Episode = 3 }; { Id = "tt1999716"; Season = 10; Episode = 4 }; { Id = "tt1999717"; Season = 10; Episode = 5 }; { Id = "tt1999718"; Season = 10; Episode = 6 }; { Id = "tt5218244"; Season = 11; Episode = 1 }; { Id = "tt5218254"; Season = 11; Episode = 2 }; { Id = "tt5218266"; Season = 11; Episode = 3 }; { Id = "tt5218284"; Season = 11; Episode = 4 }; { Id = "tt5218308"; Season = 11; Episode = 5 }; { Id = "tt5218316"; Season = 11; Episode = 6 } ] let loadTmdbJson id = let fileName = "https://raw.githubusercontent.com/ibebbs/RedDwarfAnalysis/master/TheMovieDb/" + id + ".json" JsonValue.Load(fileName) type TmdbEpisode = { Name : string; Season : int; Episode : int; AirDate : DateTime; Overview : string; } let parseTmdbJson (json : JsonValue) = { Name = json?name.AsString(); Season = json?season_number.AsInteger(); Episode = json?episode_number.AsInteger(); AirDate = json?air_date.AsDateTime(); Overview = json?overview.AsString() } "tt0684181" |> loadTmdbJson |> parseTmdbJson let ratingCategoryNames = [ "Males"; "Females"; "Aged under 18"; "Males under 18"; "Aged 18-29"; "Males Aged 18-29"; "Females Aged 18-29"; "Aged 30-44"; "Males Aged 30-44"; "Females Aged 30-44"; "Aged 45+"; "Males Aged 45+"; "Females Aged 45+"; "Top 1000 voters"; "US users"; "Non-US users"; ] type RatingCategory = | ``Males`` = 0 | ``Females`` = 1 | ``Aged under 18`` = 2 | ``Males under 18`` = 3 | ``Aged 18-29`` = 4 | ``Males Aged 18-29`` = 5 | ``Females Aged 18-29`` = 6 | ``Aged 30-44`` = 7 | ``Males Aged 30-44`` = 8 | ``Females Aged 30-44`` = 9 | ``Aged 45`` = 10 | ``Males Aged 45`` = 11 | ``Females Aged 45`` = 12 | ``Top 1000 voters`` = 13 | ``US users`` = 14 | ``Non-US users`` = 15 type EpisodeRatings = { Id : string; Category : RatingCategory; Votes : int; Rating : decimal } let parseCategory c = let index = Seq.tryFindIndex (fun cn -> cn = c) ratingCategoryNames match index with | Some x -> Some (enum(x)) | None -> None let parseRatings id = let title (node : HtmlNode) = node.Descendants["a"] |> Seq.map (fun d -> d.InnerText()) let votes (node : HtmlNode) = [ node.InnerText() ] let rating (node : HtmlNode) = [ node.InnerText() ] let document = HtmlDocument.Load("https://raw.githubusercontent.com/ibebbs/RedDwarfAnalysis/master/Ratings/" + id + ".html") let content = document.CssSelect("#tn15content").[0] let tables = content.Descendants["table"] |> Seq.toArray let rows = tables.[1].Descendants["tr"] |> Seq.map (fun row -> (row, row.Descendants["td"] |> Seq.toArray)) |> Seq.where (fun (row, data) -> data.Length = 3) |> Seq.map (fun (row, data) -> ( (title data.[0]), (votes data.[1]), (rating data.[2]))) |> Seq.collect (fun (t, v, r) -> Seq.zip3 t v r) |> Seq.map (fun (t, v, r) -> ((parseCategory t), System.Int32.Parse(v.Trim()), System.Decimal.Parse(r.Trim()))) |> Seq.where (fun (t, v, r) -> t.IsSome) |> Seq.map (fun (t, v, r) -> { Id = id; Category = t.Value; Votes = v; Rating = r }) rows type EpisodeRating = { ``Males`` : decimal option; ``Females`` : decimal option; ``Aged under 18`` : decimal option; ``Males under 18`` : decimal option; ``Aged 18-29`` : decimal option; ``Males Aged 18-29`` : decimal option; ``Females Aged 18-29`` : decimal option; ``Aged 30-44`` : decimal option; ``Males Aged 30-44`` : decimal option; ``Females Aged 30-44`` : decimal option; ``Aged 45`` : decimal option; ``Males Aged 45`` : decimal option; ``Females Aged 45`` : decimal option; ``Top 1000 voters`` : decimal option; ``US users`` : decimal option; ``Non-US users`` : decimal option; } let tryFind (dict : System.Collections.Generic.IDictionary<'a,'b>) (key : 'a) = let containsKey = dict.ContainsKey(key) match containsKey with | true -> Some dict.[key] | false -> None let pivotRatings (ratings : EpisodeRatings seq) = let dictionary = ratings |> Seq.map (fun r -> (r.Category, r.Rating)) |> dict let rating = { ``Males`` = (tryFind dictionary RatingCategory.``Males``); ``Females`` = (tryFind dictionary RatingCategory.``Females``); ``Aged under 18`` = (tryFind dictionary RatingCategory.``Aged under 18``); ``Males under 18`` = (tryFind dictionary RatingCategory.``Males under 18``); ``Aged 18-29`` = (tryFind dictionary RatingCategory.``Aged 18-29``); ``Males Aged 18-29`` = (tryFind dictionary RatingCategory.``Males Aged 18-29``); ``Females Aged 18-29`` = (tryFind dictionary RatingCategory.``Females Aged 18-29``); ``Aged 30-44`` = (tryFind dictionary RatingCategory.``Aged 30-44``); ``Males Aged 30-44`` = (tryFind dictionary RatingCategory.``Males Aged 30-44``); ``Females Aged 30-44`` = (tryFind dictionary RatingCategory.``Females Aged 30-44``); ``Aged 45`` = (tryFind dictionary RatingCategory.``Aged 45``); ``Males Aged 45`` = (tryFind dictionary RatingCategory.``Males Aged 45``); ``Females Aged 45`` = (tryFind dictionary RatingCategory.``Females Aged 45``); ``Top 1000 voters`` = (tryFind dictionary RatingCategory.``Top 1000 voters``); ``US users`` = (tryFind dictionary RatingCategory.``US users``); ``Non-US users`` = (tryFind dictionary RatingCategory.``Non-US users``) } rating let loadRatings id = let ratings = parseRatings id let rating = pivotRatings ratings rating loadRatings "tt0684181" let loadData id = let episode = id |> loadTmdbJson |> parseTmdbJson let ratings = id |> loadRatings (episode, ratings) let ratingsByDate = episodeInformation |> Seq.map (fun ei -> loadData ei.Id) |> Seq.map (fun (episode, ratings) -> (episode.AirDate, ratings.``Top 1000 voters``)) |> Seq.where (fun (date, rating) -> rating.IsSome) |> Seq.map (fun (date, rating) -> (date, rating.Value)) |> Seq.sortBy (fun (date, rating) -> date) |> Seq.toList let options = Options(pointSize=3, colors=[|"#3B8FCC"|], trendlines=[|Trendline(opacity=0.5,lineWidth=5,color="#C0D9EA")|], hAxis=Axis(title="Date"), vAxis=Axis(title="Rating")) Chart.Scatter(ratingsByDate) |> Chart.WithOptions (options) let ratingsByDateAndAgeCategory = episodeInformation |> Seq.map (fun ei -> loadData ei.Id) |> Seq.collect (fun (episode, ratings) -> [| (episode.AirDate, RatingCategory.``Aged under 18``, ratings.``Aged under 18``); (episode.AirDate, RatingCategory.``Aged 18-29``, ratings.``Aged 18-29``); (episode.AirDate, RatingCategory.``Aged 30-44``, ratings.``Aged 30-44``); (episode.AirDate, RatingCategory.``Aged 45``, ratings.``Aged 45``)|]) |> Seq.where (fun (date, category, rating) -> rating.IsSome) |> Seq.map (fun (date, category, rating) -> (date, category, rating.Value)) |> Seq.groupBy (fun (date, category, rating) -> category) |> Seq.map (fun (key, values) -> values |> Seq.map (fun (date, category, rating) -> (date, rating)) |> Seq.sortBy (fun (date, rating) -> date)) |> Seq.toList let options = Options( pointSize=3, colors=[|"#6AA590"; "#7DE6C1"; "#57E6B3"; "#60A6D0"; "#3B8FCC"|], trendlines=[| Trendline(opacity=0.5,lineWidth=5,color="#6AA590"); Trendline(opacity=0.5,lineWidth=5,color="#7DE6C1"); Trendline(opacity=0.5,lineWidth=5,color="#57E6B3"); Trendline(opacity=0.5,lineWidth=5,color="#60A6D0"); Trendline(opacity=0.5,lineWidth=5,color="#3B8FCC")|], hAxis=Axis(title="Date"), vAxis=Axis(title="Rating")) Chart.Scatter(ratingsByDateAndAgeCategory, [|"Aged under 18"; "Aged 18-29";"Aged 30-44";"Aged 45+";"????"|]) |> Chart.WithOptions(options) |> Chart.WithLegend(true)