Code

# 1. regenerate parquet + xlsx + dictionary + manifest, and assert both ways
python tools/build_web_package.py --pkg /path/to/Myrdal_Web_v1.0.0

# 2. read the counts back out of the manifest the build just wrote
import json
m = json.load(open("/path/to/Myrdal_Web_v1.0.0/WEB_MANIFEST.json", encoding="utf-8"))
c = m["counts"]
assert c["series_registered"] == c["series_published"] == c["chopped_csv"]
assert c["chopped_parquet"] == c["chopped_xlsx"] == c["chopped_csv"]
# these three are DELIBERATELY smaller -- do not "fix" them to match
print(c["series_with_data"], c["series_chartable"], c["observations_total"])

# 1. regenerate parquet + xlsx + dictionary + manifest (the builder is Python)
system("python tools/build_web_package.py --pkg /path/to/Myrdal_Web_v1.0.0")

# 2. read the counts back out of the manifest the build just wrote
library(jsonlite)
m <- fromJSON("/path/to/Myrdal_Web_v1.0.0/WEB_MANIFEST.json")
c <- m$counts
stopifnot(c$series_registered == c$series_published,
          c$series_published  == c$chopped_csv,
          c$chopped_parquet   == c$chopped_xlsx)
# these three are DELIBERATELY smaller -- do not "fix" them to match
c(c$series_with_data, c$series_chartable, c$observations_total)

import pandas as pd

# Parquet (preferred) or CSV -- identical rows.
d = pd.read_parquet("https://population.heterodata.org/downloads/series/D001.parquet")

# D001 carries TWO measures: the sum over individual countries and the
# publisher's own world aggregate. They differ; both are kept.
print(d[["year", "pop_persons", "wld_official_total"]].tail())

# The pre-1960 rows come from a different publisher than the rest. The seam is
# recorded IN THE FILE, in the `source` column -- never assume one series.
print(d["source"].unique())

library(arrow)   # or readr::read_csv for the CSV
library(dplyr)

# Parquet (preferred) or CSV -- identical rows.
d <- read_parquet("https://population.heterodata.org/downloads/series/D001.parquet")

# D001 carries TWO measures: the sum over individual countries and the
# publisher's own world aggregate. They differ; both are kept.
d %>% select(year, pop_persons, wld_official_total) %>% tail()

# The pre-1960 rows come from a different publisher than the rest. The seam is
# recorded IN THE FILE, in the `source` column -- never assume one series.
unique(d$source)