#load "Paket.fsx"
Paket.Dependencies.Install """
frameworks: net45
source https://nuget.org/api/v2
nuget FSharp.Data
nuget XPlot.GoogleCharts
"""
Paket.LoadingScripts.ScriptGeneration.generateScriptsForRootFolder
Paket.LoadingScripts.ScriptGeneration.FSharp
(Paket.FrameworkIdentifier.DotNetFramework Paket.FrameworkVersion.V4_5)
(System.IO.DirectoryInfo __SOURCE_DIRECTORY__)
#load "paket-files/include-scripts/net45/include.main.group.fsx"
//#load "XPlot.Plotly.Paket.fsx"
//#load "XPlot.Plotly.fsx"
open System
open System.IO
open FSharp.Data
open FSharp.Data.JsonExtensions
open XPlot
open XPlot.GoogleCharts
open IfSharp.Kernel.App
@"" |> Util.Html |> Display
type XPlot.GoogleCharts.GoogleChart with
member __.GetContentHtml() =
let html = __.GetInlineHtml()
html
.Replace ("google.setOnLoadCallback(drawChart);", "google.load('visualization', '1.0', { packages: ['corechart'], callback: drawChart })")
type XPlot.GoogleCharts.Chart with
static member Content (chart : GoogleChart) =
{ ContentType = "text/html"; Data = chart.GetContentHtml() }
AddDisplayPrinter (fun (plot: XPlot.GoogleCharts.GoogleChart) -> { ContentType = "text/html"; Data = plot.GetContentHtml() })
type EpisodeInformation = {
Id : string;
Season : int
Episode : int
}
let episodeInformation = [
{ Id = "tt0684181"; Season = 1; Episode = 1 };
{ Id = "tt0684157"; Season = 1; Episode = 2 };
{ Id = "tt0684145"; Season = 1; Episode = 3 };
{ Id = "tt0684186"; Season = 1; Episode = 4 };
{ Id = "tt0684151"; Season = 1; Episode = 5 };
{ Id = "tt0684165"; Season = 1; Episode = 6 };
{ Id = "tt0684161"; Season = 2; Episode = 1 };
{ Id = "tt0684146"; Season = 2; Episode = 2 };
{ Id = "tt0684180"; Season = 2; Episode = 3 };
{ Id = "tt0684177"; Season = 2; Episode = 4 };
{ Id = "tt0684175"; Season = 2; Episode = 5 };
{ Id = "tt0684169"; Season = 2; Episode = 6 };
{ Id = "tt0684144"; Season = 3; Episode = 1 };
{ Id = "tt0767232"; Season = 3; Episode = 2 };
{ Id = "tt0684172"; Season = 3; Episode = 3 };
{ Id = "tt0684148"; Season = 3; Episode = 4 };
{ Id = "tt0684185"; Season = 3; Episode = 5 };
{ Id = "tt0684183"; Season = 3; Episode = 6 };
{ Id = "tt0684149"; Season = 4; Episode = 1 };
{ Id = "tt0684152"; Season = 4; Episode = 2 };
{ Id = "tt0684160"; Season = 4; Episode = 3 };
{ Id = "tt0684187"; Season = 4; Episode = 4 };
{ Id = "tt0684153"; Season = 4; Episode = 5 };
{ Id = "tt0684164"; Season = 4; Episode = 6 };
{ Id = "tt0684159"; Season = 5; Episode = 1 };
{ Id = "tt0684182"; Season = 5; Episode = 2 };
{ Id = "tt0684179"; Season = 5; Episode = 3 };
{ Id = "tt0684174"; Season = 5; Episode = 4 };
{ Id = "tt0756588"; Season = 5; Episode = 5 };
{ Id = "tt0684143"; Season = 5; Episode = 6 };
{ Id = "tt0684173"; Season = 6; Episode = 1 };
{ Id = "tt0684163"; Season = 6; Episode = 2 };
{ Id = "tt0684158"; Season = 6; Episode = 3 };
{ Id = "tt0684155"; Season = 6; Episode = 4 };
{ Id = "tt0684176"; Season = 6; Episode = 5 };
{ Id = "tt0756589"; Season = 6; Episode = 6 };
{ Id = "tt0684184"; Season = 7; Episode = 1 };
{ Id = "tt0684178"; Season = 7; Episode = 2 };
{ Id = "tt0684168"; Season = 7; Episode = 3 };
{ Id = "tt0684154"; Season = 7; Episode = 4 };
{ Id = "tt0756587"; Season = 7; Episode = 5 };
{ Id = "tt0684147"; Season = 7; Episode = 6 };
{ Id = "tt0684156"; Season = 7; Episode = 7 };
{ Id = "tt0684166"; Season = 7; Episode = 8 };
{ Id = "tt0684140"; Season = 8; Episode = 1 };
{ Id = "tt0684141"; Season = 8; Episode = 2 };
{ Id = "tt0684142"; Season = 8; Episode = 3 };
{ Id = "tt0684150"; Season = 8; Episode = 4 };
{ Id = "tt0684162"; Season = 8; Episode = 5 };
{ Id = "tt0684170"; Season = 8; Episode = 6 };
{ Id = "tt0684171"; Season = 8; Episode = 7 };
{ Id = "tt0684167"; Season = 8; Episode = 8 };
{ Id = "tt1365540"; Season = 9; Episode = 1 };
{ Id = "tt1371606"; Season = 9; Episode = 2 };
{ Id = "tt1400975"; Season = 9; Episode = 3 };
{ Id = "tt1997038"; Season = 10; Episode = 1 };
{ Id = "tt1999714"; Season = 10; Episode = 2 };
{ Id = "tt1999715"; Season = 10; Episode = 3 };
{ Id = "tt1999716"; Season = 10; Episode = 4 };
{ Id = "tt1999717"; Season = 10; Episode = 5 };
{ Id = "tt1999718"; Season = 10; Episode = 6 };
{ Id = "tt5218244"; Season = 11; Episode = 1 };
{ Id = "tt5218254"; Season = 11; Episode = 2 };
{ Id = "tt5218266"; Season = 11; Episode = 3 };
{ Id = "tt5218284"; Season = 11; Episode = 4 };
{ Id = "tt5218308"; Season = 11; Episode = 5 };
{ Id = "tt5218316"; Season = 11; Episode = 6 }
]
let loadTmdbJson id =
let fileName = "https://raw.githubusercontent.com/ibebbs/RedDwarfAnalysis/master/TheMovieDb/" + id + ".json"
JsonValue.Load(fileName)
type TmdbEpisode = {
Name : string;
Season : int;
Episode : int;
AirDate : DateTime;
Overview : string;
}
let parseTmdbJson (json : JsonValue) = {
Name = json?name.AsString();
Season = json?season_number.AsInteger();
Episode = json?episode_number.AsInteger();
AirDate = json?air_date.AsDateTime();
Overview = json?overview.AsString()
}
"tt0684181" |> loadTmdbJson |> parseTmdbJson
let ratingCategoryNames = [
"Males";
"Females";
"Aged under 18";
"Males under 18";
"Aged 18-29";
"Males Aged 18-29";
"Females Aged 18-29";
"Aged 30-44";
"Males Aged 30-44";
"Females Aged 30-44";
"Aged 45+";
"Males Aged 45+";
"Females Aged 45+";
"Top 1000 voters";
"US users";
"Non-US users";
]
type RatingCategory =
| ``Males`` = 0
| ``Females`` = 1
| ``Aged under 18`` = 2
| ``Males under 18`` = 3
| ``Aged 18-29`` = 4
| ``Males Aged 18-29`` = 5
| ``Females Aged 18-29`` = 6
| ``Aged 30-44`` = 7
| ``Males Aged 30-44`` = 8
| ``Females Aged 30-44`` = 9
| ``Aged 45`` = 10
| ``Males Aged 45`` = 11
| ``Females Aged 45`` = 12
| ``Top 1000 voters`` = 13
| ``US users`` = 14
| ``Non-US users`` = 15
type EpisodeRatings = {
Id : string;
Category : RatingCategory;
Votes : int;
Rating : decimal
}
let parseCategory c =
let index = Seq.tryFindIndex (fun cn -> cn = c) ratingCategoryNames
match index with
| Some x -> Some (enum(x))
| None -> None
let parseRatings id =
let title (node : HtmlNode) =
node.Descendants["a"]
|> Seq.map (fun d -> d.InnerText())
let votes (node : HtmlNode) =
[ node.InnerText() ]
let rating (node : HtmlNode) =
[ node.InnerText() ]
let document = HtmlDocument.Load("https://raw.githubusercontent.com/ibebbs/RedDwarfAnalysis/master/Ratings/" + id + ".html")
let content = document.CssSelect("#tn15content").[0]
let tables =
content.Descendants["table"]
|> Seq.toArray
let rows =
tables.[1].Descendants["tr"]
|> Seq.map (fun row -> (row, row.Descendants["td"] |> Seq.toArray))
|> Seq.where (fun (row, data) -> data.Length = 3)
|> Seq.map (fun (row, data) -> ( (title data.[0]), (votes data.[1]), (rating data.[2])))
|> Seq.collect (fun (t, v, r) -> Seq.zip3 t v r)
|> Seq.map (fun (t, v, r) -> ((parseCategory t), System.Int32.Parse(v.Trim()), System.Decimal.Parse(r.Trim())))
|> Seq.where (fun (t, v, r) -> t.IsSome)
|> Seq.map (fun (t, v, r) -> { Id = id; Category = t.Value; Votes = v; Rating = r })
rows
type EpisodeRating = {
``Males`` : decimal option;
``Females`` : decimal option;
``Aged under 18`` : decimal option;
``Males under 18`` : decimal option;
``Aged 18-29`` : decimal option;
``Males Aged 18-29`` : decimal option;
``Females Aged 18-29`` : decimal option;
``Aged 30-44`` : decimal option;
``Males Aged 30-44`` : decimal option;
``Females Aged 30-44`` : decimal option;
``Aged 45`` : decimal option;
``Males Aged 45`` : decimal option;
``Females Aged 45`` : decimal option;
``Top 1000 voters`` : decimal option;
``US users`` : decimal option;
``Non-US users`` : decimal option;
}
let tryFind (dict : System.Collections.Generic.IDictionary<'a,'b>) (key : 'a) =
let containsKey = dict.ContainsKey(key)
match containsKey with
| true -> Some dict.[key]
| false -> None
let pivotRatings (ratings : EpisodeRatings seq) =
let dictionary =
ratings
|> Seq.map (fun r -> (r.Category, r.Rating))
|> dict
let rating = {
``Males`` = (tryFind dictionary RatingCategory.``Males``);
``Females`` = (tryFind dictionary RatingCategory.``Females``);
``Aged under 18`` = (tryFind dictionary RatingCategory.``Aged under 18``);
``Males under 18`` = (tryFind dictionary RatingCategory.``Males under 18``);
``Aged 18-29`` = (tryFind dictionary RatingCategory.``Aged 18-29``);
``Males Aged 18-29`` = (tryFind dictionary RatingCategory.``Males Aged 18-29``);
``Females Aged 18-29`` = (tryFind dictionary RatingCategory.``Females Aged 18-29``);
``Aged 30-44`` = (tryFind dictionary RatingCategory.``Aged 30-44``);
``Males Aged 30-44`` = (tryFind dictionary RatingCategory.``Males Aged 30-44``);
``Females Aged 30-44`` = (tryFind dictionary RatingCategory.``Females Aged 30-44``);
``Aged 45`` = (tryFind dictionary RatingCategory.``Aged 45``);
``Males Aged 45`` = (tryFind dictionary RatingCategory.``Males Aged 45``);
``Females Aged 45`` = (tryFind dictionary RatingCategory.``Females Aged 45``);
``Top 1000 voters`` = (tryFind dictionary RatingCategory.``Top 1000 voters``);
``US users`` = (tryFind dictionary RatingCategory.``US users``);
``Non-US users`` = (tryFind dictionary RatingCategory.``Non-US users``)
}
rating
let loadRatings id =
let ratings = parseRatings id
let rating = pivotRatings ratings
rating
loadRatings "tt0684181"
let loadData id =
let episode = id |> loadTmdbJson |> parseTmdbJson
let ratings = id |> loadRatings
(episode, ratings)
let ratingsByDate =
episodeInformation
|> Seq.map (fun ei -> loadData ei.Id)
|> Seq.map (fun (episode, ratings) -> (episode.AirDate, ratings.``Top 1000 voters``))
|> Seq.where (fun (date, rating) -> rating.IsSome)
|> Seq.map (fun (date, rating) -> (date, rating.Value))
|> Seq.sortBy (fun (date, rating) -> date)
|> Seq.toList
let options = Options(pointSize=3, colors=[|"#3B8FCC"|], trendlines=[|Trendline(opacity=0.5,lineWidth=5,color="#C0D9EA")|], hAxis=Axis(title="Date"), vAxis=Axis(title="Rating"))
Chart.Scatter(ratingsByDate) |> Chart.WithOptions (options)
let ratingsByDateAndAgeCategory =
episodeInformation
|> Seq.map (fun ei -> loadData ei.Id)
|> Seq.collect (fun (episode, ratings) -> [| (episode.AirDate, RatingCategory.``Aged under 18``, ratings.``Aged under 18``); (episode.AirDate, RatingCategory.``Aged 18-29``, ratings.``Aged 18-29``); (episode.AirDate, RatingCategory.``Aged 30-44``, ratings.``Aged 30-44``); (episode.AirDate, RatingCategory.``Aged 45``, ratings.``Aged 45``)|])
|> Seq.where (fun (date, category, rating) -> rating.IsSome)
|> Seq.map (fun (date, category, rating) -> (date, category, rating.Value))
|> Seq.groupBy (fun (date, category, rating) -> category)
|> Seq.map (fun (key, values) -> values |> Seq.map (fun (date, category, rating) -> (date, rating)) |> Seq.sortBy (fun (date, rating) -> date))
|> Seq.toList
let options =
Options(
pointSize=3,
colors=[|"#6AA590"; "#7DE6C1"; "#57E6B3"; "#60A6D0"; "#3B8FCC"|],
trendlines=[|
Trendline(opacity=0.5,lineWidth=5,color="#6AA590");
Trendline(opacity=0.5,lineWidth=5,color="#7DE6C1");
Trendline(opacity=0.5,lineWidth=5,color="#57E6B3");
Trendline(opacity=0.5,lineWidth=5,color="#60A6D0");
Trendline(opacity=0.5,lineWidth=5,color="#3B8FCC")|],
hAxis=Axis(title="Date"),
vAxis=Axis(title="Rating"))
Chart.Scatter(ratingsByDateAndAgeCategory, [|"Aged under 18"; "Aged 18-29";"Aged 30-44";"Aged 45+";"????"|])
|> Chart.WithOptions(options)
|> Chart.WithLegend(true)