import re
from pathlib import Path
import arrow
import pandas as pd
import requests
from bs4 import BeautifulSoup
from IPython.display import HTML, display
ead_url = "https://archivescollection.anu.edu.au/index.php/or59j;ead?sf_format=xml"
cloudstor_url = "https://cloudstor.aarnet.edu.au/plus/s/i02k4gxeEpMAUkm"
# Get the list of dates
df_dates = pd.read_csv("complete_date_list.csv", parse_dates=["date"])
# Get the list of pages
df_pages = pd.read_csv("complete_page_list.csv", parse_dates=["date"])
# Get the list of volumes
df_vols = pd.read_csv("series_list.csv")
# Extract volume number
df_vols["vol_num"] = df_vols["Item_number"].str.extract(r"-(\d+)$").astype("int")
# Get the list of volumes from ATOM in XML
response = requests.get(ead_url)
soup = BeautifulSoup(response.text)
for vol in soup.find_all("c"):
vol_num = int(re.search(r"-(\d+)$", vol.find("unitid").text).group(1).strip())
atom_url = f"https://archivescollection.anu.edu.au/index.php/sydney-stock-exchange-stock-and-share-lists-{vol_num + 1}"
# Get links to metadata and PDF in DSpace
try:
dspace_url = (
re.search(r"(http.*)$", vol.find("altformavail").get_text())
.group(1)
.strip()
)
except AttributeError:
dspace_url = None
pdf_url = None
else:
pdf_url = f'{dspace_url.replace("handle", "bitstream")}/2/AU%20NBAC%20N193-{vol_num:03}.pdf'
# Create link to download folder of images from Cludstor
cloudstor_folder_url = (
f"{cloudstor_url}/download?path=AU%20NBAC%20N193-{vol_num:03}"
)
# Get the list of pages from this volume and save as a CSV
pages = df_pages.loc[df_pages["vol_num"] == vol_num].copy(deep=False)
if not pages.empty:
# Add a link to the page image on Cloudstor
pages["image_url"] = pages.apply(
lambda x: f'{cloudstor_folder_url}&files=N193-{int(vol_num):03}_{int(x["page_num"]):04}.jpg',
axis=1,
)
csv_file = Path(f"volumes/vol-{vol_num}-pages.csv")
pages.to_csv(csv_file, index=False)
else:
csv_file = None
# Get the date range of this volume
vol_details = df_vols.loc[df_vols["vol_num"] == vol_num]
start_date = vol_details.iloc[0]["start_date"]
end_date = vol_details.iloc[0]["end_date"]
# Get list of dates covered by this volume
dates = df_dates.loc[
(df_dates["date"] >= start_date) & (df_dates["date"] <= end_date)
]
# Get total page numbers
num_pages = dates["pages"].sum()
num_expected = dates["expected"].sum()
# Display the volume details
display(
HTML(
f'<h3>Volume {vol_num}: {arrow.get(start_date).format("D MMMM YYYY")} – {arrow.get(end_date).format("D MMMM YYYY")}</h3>'
)
)
display(
HTML(
f"<p>Estimated {num_pages} of {num_expected} pages ({num_pages / num_expected:.2%} complete)</p>"
)
)
links = f'<ul><li><a href="{atom_url}">View description in archive</a></li>'
if dspace_url:
links += f'<li><a href="{dspace_url}">View item in respository</a></li>'
links += f'<li><a href="{pdf_url}">View PDF</a></li>'
links += (
f'<li><a href="{cloudstor_folder_url}">Download images from Cloudstor</a></li>'
)
if csv_file:
links += f'<li><a href="{csv_file}" download="{csv_file}">Download page list as CSV</a></li>'
links += "</ul>"
display(HTML(links))
Estimated 303 of 317 pages (95.58% complete)
Estimated 312 of 314 pages (99.36% complete)
Estimated 345 of 346 pages (99.71% complete)
Estimated 312 of 319 pages (97.81% complete)
Estimated 305 of 312 pages (97.76% complete)
Estimated 334 of 341 pages (97.95% complete)
Estimated 349 of 351 pages (99.43% complete)
Estimated 317 of 319 pages (99.37% complete)
Estimated 327 of 329 pages (99.39% complete)
Estimated 327 of 329 pages (99.39% complete)
Estimated 350 of 351 pages (99.72% complete)
Estimated 310 of 319 pages (97.18% complete)
Estimated 330 of 334 pages (98.80% complete)
Estimated 329 of 329 pages (100.00% complete)
Estimated 349 of 351 pages (99.43% complete)
Estimated 312 of 314 pages (99.36% complete)
Estimated 329 of 329 pages (100.00% complete)
Estimated 322 of 324 pages (99.38% complete)
Estimated 348 of 348 pages (100.00% complete)
Estimated 312 of 312 pages (100.00% complete)
Estimated 330 of 331 pages (99.70% complete)
Estimated 314 of 324 pages (96.91% complete)
Estimated 344 of 346 pages (99.42% complete)
Estimated 310 of 312 pages (99.36% complete)
Estimated 323 of 327 pages (98.78% complete)
Estimated 332 of 336 pages (98.81% complete)
Estimated 349 of 351 pages (99.43% complete)
Estimated 314 of 319 pages (98.43% complete)
Estimated 328 of 329 pages (99.70% complete)
Estimated 327 of 329 pages (99.39% complete)
Estimated 339 of 341 pages (99.41% complete)
Estimated 316 of 319 pages (99.06% complete)
Estimated 329 of 329 pages (100.00% complete)
Estimated 322 of 324 pages (99.38% complete)
Estimated 350 of 351 pages (99.72% complete)
Estimated 317 of 319 pages (99.37% complete)
Estimated 300 of 302 pages (99.34% complete)
Estimated 328 of 331 pages (99.09% complete)
Estimated 350 of 351 pages (99.72% complete)
Estimated 317 of 319 pages (99.37% complete)
Estimated 325 of 329 pages (98.78% complete)
Estimated 318 of 322 pages (98.76% complete)
Estimated 343 of 343 pages (100.00% complete)
Estimated 317 of 317 pages (100.00% complete)
Estimated 331 of 331 pages (100.00% complete)
Estimated 317 of 319 pages (99.37% complete)
Estimated 350 of 351 pages (99.72% complete)
Estimated 314 of 314 pages (100.00% complete)
Estimated 296 of 302 pages (98.01% complete)
Estimated 337 of 341 pages (98.83% complete)
Estimated 351 of 351 pages (100.00% complete)
Estimated 313 of 317 pages (98.74% complete)
Estimated 325 of 329 pages (98.78% complete)
Estimated 315 of 319 pages (98.75% complete)
Estimated 167 of 351 pages (47.58% complete)
Estimated 315 of 319 pages (98.75% complete)
Estimated 318 of 319 pages (99.69% complete)
Estimated 309 of 309 pages (100.00% complete)
Estimated 347 of 351 pages (98.86% complete)
Estimated 313 of 319 pages (98.12% complete)
Estimated 317 of 317 pages (100.00% complete)
Estimated 312 of 314 pages (99.36% complete)
Estimated 345 of 348 pages (99.14% complete)
Estimated 309 of 313 pages (98.72% complete)
Estimated 317 of 319 pages (99.37% complete)
Estimated 307 of 314 pages (97.77% complete)
Estimated 343 of 346 pages (99.13% complete)
Estimated 313 of 317 pages (98.74% complete)
Estimated 310 of 310 pages (100.00% complete)
Estimated 325 of 326 pages (99.69% complete)
Estimated 351 of 351 pages (100.00% complete)
Estimated 296 of 302 pages (98.01% complete)
Estimated 321 of 324 pages (99.07% complete)
Estimated 314 of 314 pages (100.00% complete)
Estimated 348 of 349 pages (99.71% complete)
Estimated 306 of 309 pages (99.03% complete)
Estimated 331 of 334 pages (99.10% complete)
Estimated 309 of 309 pages (100.00% complete)
Estimated 351 of 351 pages (100.00% complete)
Estimated 317 of 319 pages (99.37% complete)
Estimated 281 of 285 pages (98.60% complete)
Estimated 333 of 336 pages (99.11% complete)
Estimated 350 of 351 pages (99.72% complete)
Estimated 312 of 314 pages (99.36% complete)
Estimated 315 of 317 pages (99.37% complete)
Estimated 307 of 309 pages (99.35% complete)
Estimated 348 of 348 pages (100.00% complete)
Estimated 315 of 317 pages (99.37% complete)
Estimated 305 of 307 pages (99.35% complete)
Estimated 316 of 321 pages (98.44% complete)
Estimated 343 of 344 pages (99.71% complete)
Estimated 319 of 319 pages (100.00% complete)
Estimated 325 of 327 pages (99.39% complete)
Estimated 301 of 304 pages (99.01% complete)
Estimated 351 of 351 pages (100.00% complete)
Estimated 317 of 317 pages (100.00% complete)
Estimated 314 of 314 pages (100.00% complete)
Estimated 308 of 317 pages (97.16% complete)
Estimated 346 of 346 pages (100.00% complete)
Estimated 317 of 319 pages (99.37% complete)
Estimated 315 of 319 pages (98.75% complete)
Estimated 314 of 314 pages (100.00% complete)
Estimated 351 of 351 pages (100.00% complete)
Estimated 313 of 314 pages (99.68% complete)
Estimated 320 of 324 pages (98.77% complete)
Estimated 312 of 314 pages (99.36% complete)
Estimated 349 of 351 pages (99.43% complete)
Estimated 309 of 309 pages (100.00% complete)
Estimated 329 of 331 pages (99.40% complete)
Estimated 310 of 314 pages (98.73% complete)
Estimated 343 of 346 pages (99.13% complete)
Estimated 317 of 317 pages (100.00% complete)
Estimated 305 of 310 pages (98.39% complete)
Estimated 326 of 326 pages (100.00% complete)
Estimated 351 of 351 pages (100.00% complete)
Estimated 314 of 314 pages (100.00% complete)
Estimated 322 of 324 pages (99.38% complete)
Estimated 307 of 312 pages (98.40% complete)
Estimated 351 of 351 pages (100.00% complete)
Estimated 317 of 319 pages (99.37% complete)
Estimated 322 of 324 pages (99.38% complete)
Estimated 314 of 317 pages (99.05% complete)
Estimated 347 of 351 pages (98.86% complete)
Estimated 316 of 319 pages (99.06% complete)
Estimated 296 of 302 pages (98.01% complete)
Estimated 340 of 341 pages (99.71% complete)
Estimated 351 of 351 pages (100.00% complete)
Estimated 312 of 314 pages (99.36% complete)
Estimated 327 of 329 pages (99.39% complete)
Estimated 307 of 314 pages (97.77% complete)
Estimated 348 of 348 pages (100.00% complete)
Estimated 315 of 317 pages (99.37% complete)
Estimated 303 of 307 pages (98.70% complete)
Estimated 309 of 313 pages (98.72% complete)
Estimated 390 of 392 pages (99.49% complete)
Estimated 365 of 368 pages (99.18% complete)
Estimated 375 of 378 pages (99.21% complete)
Estimated 366 of 366 pages (100.00% complete)
Estimated 416 of 422 pages (98.58% complete)
Estimated 371 of 372 pages (99.73% complete)
Estimated 381 of 390 pages (97.69% complete)
Estimated 373 of 376 pages (99.20% complete)
Estimated 416 of 416 pages (100.00% complete)
Estimated 366 of 370 pages (98.92% complete)
Estimated 359 of 364 pages (98.63% complete)
Estimated 605 of 606 pages (99.83% complete)
Estimated 621 of 624 pages (99.52% complete)
Estimated 564 of 567 pages (99.47% complete)
Estimated 585 of 585 pages (100.00% complete)
Estimated 555 of 558 pages (99.46% complete)
Estimated 623 of 624 pages (99.84% complete)
Estimated 555 of 558 pages (99.46% complete)
Estimated 600 of 603 pages (99.50% complete)
Estimated 573 of 585 pages (97.95% complete)
Estimated 605 of 606 pages (99.83% complete)
Estimated 558 of 564 pages (98.94% complete)
Estimated 579 of 585 pages (98.97% complete)
Estimated 606 of 606 pages (100.00% complete)
Estimated 632 of 633 pages (99.84% complete)
Estimated 555 of 558 pages (99.46% complete)
Estimated 591 of 597 pages (98.99% complete)
Estimated 570 of 585 pages (97.44% complete)
Estimated 620 of 624 pages (99.36% complete)
Estimated 600 of 606 pages (99.01% complete)
Estimated 439 of 501 pages (87.62% complete)
Estimated 408 of 411 pages (99.27% complete)
Estimated 429 of 429 pages (100.00% complete)
Estimated 414 of 414 pages (100.00% complete)
Estimated 416 of 417 pages (99.76% complete)
Estimated 408 of 408 pages (100.00% complete)
Estimated 428 of 429 pages (99.77% complete)
Estimated 392 of 396 pages (98.99% complete)
Estimated 414 of 414 pages (100.00% complete)
Estimated 393 of 396 pages (99.24% complete)
Estimated 426 of 426 pages (100.00% complete)
Estimated 384 of 387 pages (99.22% complete)
Estimated 393 of 396 pages (99.24% complete)
Estimated 399 of 399 pages (100.00% complete)
Estimated 411 of 411 pages (100.00% complete)
Estimated 384 of 387 pages (99.22% complete)
Estimated 405 of 405 pages (100.00% complete)
Estimated 378 of 384 pages (98.44% complete)
Estimated 264 of 291 pages (90.72% complete)
Estimated 387 of 390 pages (99.23% complete)
Estimated 396 of 396 pages (100.00% complete)
Estimated 387 of 390 pages (99.23% complete)
Estimated 429 of 429 pages (100.00% complete)
Estimated 386 of 390 pages (98.97% complete)
Estimated 345 of 348 pages (99.14% complete)
Estimated 378 of 378 pages (100.00% complete)
Estimated 489 of 488 pages (100.20% complete)
Estimated 468 of 472 pages (99.15% complete)
Estimated 488 of 488 pages (100.00% complete)
Estimated 468 of 472 pages (99.15% complete)
Estimated 520 of 520 pages (100.00% complete)
Estimated 460 of 464 pages (99.14% complete)
Estimated 496 of 496 pages (100.00% complete)
Estimated 460 of 464 pages (99.14% complete)
Estimated 644 of 648 pages (99.38% complete)
Created by Tim Sherratt for the GLAM Workbench.