from urllib.request import urlopen
import json
import re
from bs4 import BeautifulSoup
from datetime import datetime
import matplotlib.pyplot as plt
import requests
import numpy as np
from sklearn import preprocessing
import time
# Declare Website URL
URL = 'http://redditmetrics.com/r/'
Input: a cryptocoin ticker (e.g. 'BTC', 'ETH', etc...)
Output: three lists (dates, subscriber growth, and total subscribers)
# Input a coin and the past x days- return three lists: dates, subscriber growth, and total subscribers
def get_reddit_metrics_data(coin):
temp = coin.replace(' ', '')
url = URL + temp
# Open url with urllib
with urlopen(url) as url:
web = url.read()
# Create instance of soup and grab script tag containing data
soup = BeautifulSoup(web, 'lxml')
data = soup.find_all("script")[8].string
data = data.split()
# Parse Subscriber Growth Data
sub_growth_start = data.index('element:')
assert data[sub_growth_start + 1] == "'subscriber-growth',"
sub_growth_end = data.index("pointSize:'',")
subscriber_growth = data[sub_growth_start:sub_growth_end]
######################### SUBSCRIBER GROWTH DATA ###################
# Parse Dates and return a list of dates
date_pattern = re.compile("'\d{4}(?P<sep>[-/])\d{2}(?P=sep)\d{2}'")
dates = list(filter(date_pattern.match, subscriber_growth))
dates = [replace(x) for x in dates]
dates = [datetime.strptime(x, '%Y-%m-%d') for x in dates]
# Parse Values
value_pattern = re.compile("\d*}")
# print(subscriber_growth)
subgrowth_values = list(filter(value_pattern.match, subscriber_growth))
subgrowth_values = [replace(x) for x in subgrowth_values]
subgrowth_values = list(map(int, subgrowth_values))
################ TOTAL SUBSCRIBERS ################
# Get Total Subscribers data
data_1 = data[sub_growth_end+1:]
total_sub_start = data_1.index('element:')
assert data_1[total_sub_start + 1] == "'total-subscribers',"
total_sub_end = data_1.index("pointSize:'',")
total_subscribers = data_1[total_sub_start:total_sub_end]
# Parse Values
totalsubs_values = list(filter(value_pattern.match, total_subscribers))
totalsubs_values = [replace(x) for x in totalsubs_values]
totalsubs_values = list(map(int, totalsubs_values))
return dates, subgrowth_values, totalsubs_values
# Replace commas, quotes, and brackets with blanks
def replace(string):
x = string.replace(",", '')
y = x.replace("'", '')
z = y.replace("}", '')
return z
# Graph Dates and Values Using Matplotlib
def graph(dates, values, title, xlabel, ylabel, pastxdays = 100):
last_x_days_dates = dates[-pastxdays:]
last_x_days_values = values[-pastxdays:]
plt.figure(figsize=(10,5))
plt.title(title)
plt.xlabel(xlabel)
plt.ylabel(ylabel)
plt.plot(last_x_days_dates, last_x_days_values)
plt.show()
# Get historical data
def historicalprices(ticker, currency='USD'):
endpoint = requests.get('https://min-api.cryptocompare.com/data/histoday?aggregate=1&e=CCCAGG&extraParam'
'=CryptoCompare&fsym=' + ticker.upper() + '&limit=100000&tryConversion=false&tsym=' +
currency.upper())
endpoint = endpoint.json()
closeprice = [data['close'] for data in endpoint['Data']]
closeprice = list(map(float, closeprice))
time = [data['time'] for data in endpoint['Data']]
time = [datetime.fromtimestamp(unixtime).strftime('%Y-%m-%d') for unixtime in time]
time = [datetime.strptime(x, '%Y-%m-%d') for x in time]
return time, closeprice
if __name__ == "__main__":
### Name: Symbol
coinlist = {}
# MasterData.txt has dictionary of symbols, so we just extract the symbols from there
# Store list of coins in 'names'
json_data = json.load(open('MasterData.txt'))
key = list(json_data.keys())[0]
names = list(json_data[key].keys())
# For each coin, try to get data, otherwise pass and continue
try:
for coin in names:
coinlist[json_data[key][coin]['General']['CoinName']] = coin
except:
pass
# See which coins have failed and maybe eliminate them next time
failed = []
# Plot data
for coin in coinlist:
try:
time.sleep(3)
print(coin, coinlist[coin])
dates, subgrowth, totalsub = get_reddit_metrics_data(coin)
_, closeprice = historicalprices(coinlist[coin], 'USD')
# print(closeprice)
# graph(dates, closeprice, coin, 'Date', 'Price', pastxdays = 100)
normalized_subgrowth = preprocessing.normalize([subgrowth])
normalized_price = preprocessing.normalize([closeprice])
normalized_totalsub = preprocessing.normalize([totalsub])
plt.figure(figsize=(10,5))
plt.title(coin)
plt.plot(dates[-100:], normalized_subgrowth[0][-100:], 'r')
plt.plot(dates[-100:], normalized_price[0][-100:], 'b')
plt.plot(dates[-100:], normalized_totalsub[0][-100:], 'g')
plt.show()
except:
print('FAILED FOR: ' + coin)
failed.append(coin)
Nxt NXT
Ethereum Classic ETC
Dogecoin DOGE
FAILED FOR: Bitcoin Monero XMR
ZCash ZEC
Litecoin LTC
DigitalCash DASH
Bitshares BTS
DigiByte DGB
BitcoinDark BTCD FAILED FOR: BitcoinDark ProsperCoin PRC
Bitstake XBS
PayCoin PYC FAILED FOR: PayCoin CraigsCoin CRAIG FAILED FOR: CraigsCoin Ripple XRP
PeerCoin PPC
YbCoin YBC
Ethereum ETH
DarkKush DANK FAILED FOR: DarkKush DarkToken DT FAILED FOR: DarkToken CETUS Coin CETI FAILED FOR: CETUS Coin KoboCoin KOBO
Supcoin SUP FAILED FOR: Supcoin PetroDollar XPD
CleverHash CHASH FAILED FOR: CleverHash GiveCoin GIVE FAILED FOR: GiveCoin GeoCoin GEO
Spreadcoin SPR
Insanity Coin WOLF FAILED FOR: Insanity Coin NXTI NXTI FAILED FOR: NXTI DogeParty XDP FAILED FOR: DogeParty ACoin ACOIN
2015 coin 2015 FAILED FOR: 2015 coin AlphaCoin ALF FAILED FOR: AlphaCoin Aero Coin AERO FAILED FOR: Aero Coin Aegis AGS FAILED FOR: Aegis AmericanCoin AMC FAILED FOR: AmericanCoin AlienCoin ALN FAILED FOR: AlienCoin ApexCoin APEX FAILED FOR: ApexCoin Argentum ARG FAILED FOR: Argentum ArchCoin ARCH FAILED FOR: ArchCoin Asia Coin AC FAILED FOR: Asia Coin AXRON AXR FAILED FOR: AXRON Aurora Coin AUR
BattleCoin BCX FAILED FOR: BattleCoin Benjamins BEN FAILED FOR: Benjamins BetaCoin BET FAILED FOR: BetaCoin BitBean BITB
BoostCoin BOST
BQCoin BQC FAILED FOR: BQCoin MyriadCoin XMY FAILED FOR: MyriadCoin 42 Coin 42 FAILED FOR: 42 Coin MoonCoin MOON
SexCoin SXC
Quatloo QTL FAILED FOR: Quatloo BlueCoin BLU
BlackCoin BLK
EnergyCoin ENRG FAILED FOR: EnergyCoin QuarkCoin QRK FAILED FOR: QuarkCoin ZetaCoin ZET
Riecoin RIC FAILED FOR: Riecoin LimeCoinX LIMX FAILED FOR: LimeCoinX CAIx CAIX FAILED FOR: CAIx BitBar BTB FAILED FOR: BitBar AriCoin ARI FAILED FOR: AriCoin ByteCoin BCN FAILED FOR: ByteCoin BitMark BTM FAILED FOR: BitMark BitGem BTG* FAILED FOR: BitGem CryptoBuk BUK FAILED FOR: CryptoBuk BottleCaps CAP FAILED FOR: BottleCaps CashCoin CASH FAILED FOR: CashCoin CryptoBullion CBX
CinderCoin CIN FAILED FOR: CinderCoin CannaCoin CCN FAILED FOR: CannaCoin CINNICOIN CINNI FAILED FOR: CINNICOIN CheckCoin CXC FAILED FOR: CheckCoin CloakCoin CLOAK FAILED FOR: CloakCoin CopperLark CLR FAILED FOR: CopperLark Cachecoin CACH FAILED FOR: Cachecoin ChinaCoin CNC FAILED FOR: ChinaCoin ConcealCoin CNL FAILED FOR: ConcealCoin CoolCoin COOL FAILED FOR: CoolCoin Community Coin COC
CosmosCoin CMC
CrackCoin CRACK FAILED FOR: CrackCoin CryptCoin CRYPT FAILED FOR: CryptCoin CasinoCoin CSC FAILED FOR: CasinoCoin eMark DEM FAILED FOR: eMark DarkShibe DSB FAILED FOR: DarkShibe DarkCash DRKC FAILED FOR: DarkCash DigiCoin DGC FAILED FOR: DigiCoin DevCoin DVC FAILED FOR: DevCoin Diamond DMD
E-Gulden EFL FAILED FOR: E-Gulden EarthCoin EAC FAILED FOR: EarthCoin Elacoin ELC
Einsteinium EMC2 FAILED FOR: Einsteinium Exclusive Coin EXCL FAILED FOR: Exclusive Coin ExeCoin EXE FAILED FOR: ExeCoin EZCoin EZC FAILED FOR: EZCoin FireflyCoin FFC
CLAMS CLAM
FlutterCoin FLT FAILED FOR: FlutterCoin FireRoosterCoin FRC FAILED FOR: FireRoosterCoin Franko FRK
FractalCoin FRAC FAILED FOR: FractalCoin FeatherCoin FTC
GlobalCoin GLC FAILED FOR: GlobalCoin GoldCoin GLD
GalaxyCoin GLX FAILED FOR: GalaxyCoin GameLeagueCoin GML FAILED FOR: GameLeagueCoin Emerald EMD FAILED FOR: Emerald GuerillaCoin GUE FAILED FOR: GuerillaCoin CraftCoin CRC
Halcyon HAL FAILED FOR: Halcyon HoboNickels HBN FAILED FOR: HoboNickels HunterCoin HUC FAILED FOR: HunterCoin HeavyCoin HVC FAILED FOR: HeavyCoin Hyperstake HYP
Fuel2Coin FC2 FAILED FOR: Fuel2Coin IceBergCoin ICB FAILED FOR: IceBergCoin IOCoin IOC FAILED FOR: IOCoin Flappy Coin FLAP FAILED FOR: Flappy Coin IXcoin IXC
FIBRE FIBRE FAILED FOR: FIBRE JumBucks Coin JBS FAILED FOR: JumBucks Coin JudgeCoin JUDGE FAILED FOR: JudgeCoin KeyCoin KEY FAILED FOR: KeyCoin KrugerCoin KGC
CoinWorksCoin LAB* FAILED FOR: CoinWorksCoin Lucky7Coin LK7 FAILED FOR: Lucky7Coin GrandCoin GDC FAILED FOR: GrandCoin LightSpeedCoin LSD FAILED FOR: LightSpeedCoin Litebar LTB FAILED FOR: Litebar LitecoinDark LTCD FAILED FOR: LitecoinDark GlyphCoin GLYPH FAILED FOR: GlyphCoin LibrexCoin LXC FAILED FOR: LibrexCoin LycanCoin LYC FAILED FOR: LycanCoin MaxCoin MAX
MediterraneanCoin MED FAILED FOR: MediterraneanCoin LitecoinX LTCX FAILED FOR: LitecoinX MintCoin MINT FAILED FOR: MintCoin Infinite Coin IFC FAILED FOR: Infinite Coin MinCoin MNC FAILED FOR: MinCoin JunkCoin JKC FAILED FOR: JunkCoin Klondike Coin KDC FAILED FOR: Klondike Coin Legendary Coin LGD* FAILED FOR: Legendary Coin MysteryCoin MYST* FAILED FOR: MysteryCoin NanoToken NAN