Changes to the Trove API mean that the techniques I've previously used to select resources at random will no longer work. This notebook provides one alternative.
I wanted something that would work efficiently, but would also expose as much of the content as possible. Applying multiple facets together with a randomly-generated query seems to do a good job of getting the result set below 100 (the maximum available from a single API call). This should mean that most of the newspaper articles are reachable, but it's a bit hard to quantify.
Thanks to Mitchell Harrop for suggesting I could use randomly selected stopwords as queries. I've supplemented the stopwords with letters and digits, and together they seem to do a good job of applying an initial filter and mixing up the relevance ranking.
As you can see from the examples below, you can supply any of the facets available in the newspapers zone – for example: state
, title
, year
, illType
, category
.
import json
import os
import random
import requests
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
s = requests.Session()
retries = Retry(total=5, backoff_factor=1, status_forcelist=[502, 503, 504])
s.mount("https://", HTTPAdapter(max_retries=retries))
s.mount("http://", HTTPAdapter(max_retries=retries))
with open("stopwords.json", "r") as json_file:
STOPWORDS = json.load(json_file)
API_URL = "http://api.trove.nla.gov.au/v2/result"
%%capture
# Load variables from the .env file if it exists
# Use %%capture to suppress messages
%load_ext dotenv
%dotenv
# Insert your Trove API key
API_KEY = "YOUR API KEY"
# Use api key value from environment variables if it is available
if os.getenv("TROVE_API_KEY"):
API_KEY = os.getenv("TROVE_API_KEY")
def get_random_facet_value(params, facet):
"""
Get values for the supplied facet and choose one at random.
"""
these_params = params.copy()
these_params["facet"] = facet
response = s.get(API_URL, params=these_params)
data = response.json()
try:
values = [
t["search"] for t in data["response"]["zone"][0]["facets"]["facet"]["term"]
]
except TypeError:
return None
return random.choice(values)
def get_total_results(params):
response = s.get(API_URL, params=params)
data = response.json()
total = int(data["response"]["zone"][0]["records"]["total"])
return total
def get_random_article(query=None, **kwargs):
"""
Get a random article.
The kwargs can be any of the available facets, such as 'state', 'title', 'illtype', 'year'.
"""
total = 0
applied_facets = []
facets = ["month", "year", "decade", "word", "illustrated", "category", "title"]
tries = 0
params = {
"zone": "newspaper",
"encoding": "json",
# Note that keeping n at 0 until we've filtered the result set speeds things up considerably
"n": "0",
# Uncomment these if you need more than the basic data
# "reclevel": "full",
# 'include': 'articleText',
"key": API_KEY,
}
if query:
params["q"] = query
# If there's no query supplied then use a random stopword to mix up the results
else:
random_word = random.choice(STOPWORDS)
params["q"] = f'"{random_word}"'
# Apply any supplied factes
for key, value in kwargs.items():
params[f"l-{key}"] = value
applied_facets.append(key)
# Remove any facets that have already been applied from the list of available facets
facets[:] = [f for f in facets if f not in applied_facets]
total = get_total_results(params)
# If our randomly selected stopword has produced no results
# keep trying with new queries until we get some (give up after 10 tries)
while total == 0 and tries <= 10:
if not query:
random_word = random.choice(STOPWORDS)
params["q"] = f'"{random_word}"'
tries += 1
# Apply facets one at a time until we have less than 100 results, or we run out of facets
while total > 100 and len(facets) > 0:
# Get the next facet
facet = facets.pop()
# Set the facet to a randomly selected value
params[f"l-{facet}"] = get_random_facet_value(params, facet)
total = get_total_results(params)
# print(total)
# print(response.url)
# If we've ended up with some results, then select one (of the first 100) at random
if total > 0:
params["n"] = "100"
response = s.get(API_URL, params=params)
data = response.json()
article = random.choice(data["response"]["zone"][0]["records"]["article"])
return article
get_random_article()
{'id': '153441823', 'url': '/newspaper/153441823', 'heading': 'Advertising', 'category': 'Advertising', 'title': {'id': '765', 'value': 'Every Week (Bairnsdale, Vic. : 1914 - 1918)'}, 'date': '1917-02-15', 'page': 4, 'pageSequence': 4, 'relevance': {'score': '0.105540276', 'value': 'may have relevance'}, 'troveUrl': 'https://trove.nla.gov.au/ndp/del/article/153441823?searchTerm=%22on%22'}
get_random_article(query="pademelon")
{'id': '67402600', 'url': '/newspaper/67402600', 'heading': 'THE BUCHAN CAVES. BEAUTIFUL NATURE. EQUAL TO JENOLAN.', 'category': 'Article', 'title': {'id': '209', 'value': 'The Maffra Spectator (Vic. : 1882 - 1920)'}, 'date': '1907-12-30', 'page': 3, 'pageSequence': 3, 'relevance': {'score': '7.7346535', 'value': 'very relevant'}, 'snippet': '"Where will we go for the holidays?" is the usual question that has to be answered by people at this time of the year, and in due course they proceed', 'troveUrl': 'https://trove.nla.gov.au/ndp/del/article/67402600?searchTerm=pademelon'}
get_random_article(state="Tasmania")
{'id': '232486642', 'url': '/newspaper/232486642', 'heading': 'Advertising', 'category': 'Advertising', 'title': {'id': '1236', 'value': 'The Cornwall Press and Commercial Advertiser (Launceston, Tas. : 1829)'}, 'date': '1829-03-17', 'page': 4, 'pageSequence': 4, 'relevance': {'score': '0.052769434', 'value': 'may have relevance'}, 'troveUrl': 'https://trove.nla.gov.au/ndp/del/article/232486642?searchTerm=%22to%22'}
get_random_article(title="35", category="Article")
{'id': '14222939', 'url': '/newspaper/14222939', 'heading': 'SEARCHING FOR THE PERTHSHIRE. RETURN OF THE TUGBOAT HERO. WAS WITHIN TWENTY MILES OF THE DISABLED SHIP.', 'category': 'Article', 'title': {'id': '35', 'value': 'The Sydney Morning Herald (NSW : 1842 - 1954)'}, 'date': '1899-06-03', 'page': 9, 'pageSequence': 9, 'relevance': {'score': '6.375327', 'value': 'very relevant'}, 'snippet': "Anothor vessel reached port yesterday after a 12 days' cruise in search of tho Perthshire. She was the tug Hero, belonging to Messrs J Fenwick and Co, and as she lay at the Floating Jetty, Circular Quay,", 'troveUrl': 'https://trove.nla.gov.au/ndp/del/article/14222939?searchTerm=%22hadn%22'}
get_random_article(illustrated="true")
{'id': '258155681', 'url': '/newspaper/258155681', 'heading': 'SMARTLY RESCUED. A SYDNEY LADY IN PERIL.', 'category': 'Article', 'title': {'id': '1586', 'value': 'The Nowra Colonist (NSW : 1899 - 1904)'}, 'date': '1899-09-06', 'page': 4, 'pageSequence': 4, 'relevance': {'score': '1.7316521', 'value': 'likely to be relevant'}, 'snippet': 'The following particulars just to ha[?] of a smart rescue effected under difficulties should be of interest to must [?] Mrs David [?] of 257', 'troveUrl': 'https://trove.nla.gov.au/ndp/del/article/258155681?searchTerm=%22p%22'}
get_random_article(title="112", illustrated="true", category="Advertising")
{'id': '47119929', 'url': '/newspaper/47119929', 'heading': 'Advertising', 'category': 'Advertising', 'title': {'id': '112', 'value': "The Australian Women's Weekly (1933 - 1982)"}, 'date': '1957-06-12', 'page': 11, 'pageSequence': 11, 'relevance': {'score': '0.43551445', 'value': 'likely to be relevant'}, 'troveUrl': 'https://trove.nla.gov.au/ndp/del/article/47119929?searchTerm=%22own%22'}
get_random_article(illtype="Cartoon")
{'id': '159624812', 'url': '/newspaper/159624812', 'heading': 'ALL DOUBTS REMOVED.', 'category': 'Article', 'title': {'id': '644', 'value': 'The Gloucester Advocate (NSW : 1905 - 1954)'}, 'date': '1929-05-21', 'page': 4, 'pageSequence': 4, 'relevance': {'score': '15.173103', 'value': 'very relevant'}, 'snippet': 'THE BOSS: "Smoke cigars, Tompkins?" TOMPKINS: "Yes, sir. I\'m very partial to a good cigar." THE BOSS: "Umph—then I\'ll lock \'em up."', 'troveUrl': 'https://trove.nla.gov.au/ndp/del/article/159624812?searchTerm=%22up%22'}
get_random_article(year="1930")
{'id': '143816816', 'url': '/newspaper/143816816', 'heading': 'Political Activities of the C.P. Campaign in Murray', 'category': 'Article', 'title': {'id': '655', 'value': 'Riverina Recorder (Balranald, Moulamein, NSW : 1887 - 1944)'}, 'date': '1930-09-27', 'page': 3, 'pageSequence': 3, 'relevance': {'score': '1.1914315', 'value': 'likely to be relevant'}, 'snippet': 'An early review reveals a substantial foundation for high hopes in the prospects of a Country Party representative being returned, but hard work', 'troveUrl': 'https://trove.nla.gov.au/ndp/del/article/143816816?searchTerm=%22g%22'}
get_random_article(publictag="poem")
{'id': '63224945', 'url': '/newspaper/63224945', 'heading': 'Sketches with Pen THE PREVISION OF JOHN BROWN.', 'category': 'Article', 'title': {'id': '49', 'value': 'The Australasian Sketcher with Pen and Pencil (Melbourne, Vic. : 1873 - 1889)'}, 'date': '1889-05-16', 'page': 67, 'pageSequence': 67, 'relevance': {'score': '1.7888632', 'value': 'likely to be relevant'}, 'snippet': 'Brown was weeping likewise cursing; and with amplitude of reason, For a letter had been handed him that very afternoon Which proved he had been cruelly begotten out of season', 'troveUrl': 'https://trove.nla.gov.au/ndp/del/article/63224945?searchTerm=%223%22'}
%%timeit
get_random_article()
The slowest run took 4.60 times longer than the fastest. This could mean that an intermediate result is being cached. 4.92 s ± 2.6 s per loop (mean ± std. dev. of 7 runs, 1 loop each)
Created by Tim Sherratt for the GLAM Workbench.