Downloads¶

# Latest monthly full bundle (JSON)
curl -O https://pub-f0ad0e0dac0345408dcc95bda20beb42.r2.dev/datasets/clinvar-gkm_00-latest.json.gz

# Latest weekly delta + its manifest
curl -O https://pub-f0ad0e0dac0345408dcc95bda20beb42.r2.dev/deltas/00-latest/clinvar-gkm-delta_00-latest.json.gz
curl -O https://pub-f0ad0e0dac0345408dcc95bda20beb42.r2.dev/deltas/00-latest/manifest.json

# Decompress
gunzip clinvar-gkm_00-latest.json.gz

# Download a single Parquet section from the latest monthly full (e.g., SCV statements)
curl -O https://pub-f0ad0e0dac0345408dcc95bda20beb42.r2.dev/datasets/parquet/00-latest/scv.parquet

# Download all Parquet files (latest monthly full)
for section in sequenceReference location allele copyNumberCount copyNumberChange \
               gene variation condition conditionSet therapy therapyGroup submitter \
               varcond-proposition vartumor-proposition vartherapy-proposition varcustom-proposition \
               evidenceLine vcv_evidenceLine rcv_evidenceLine \
               scv vcv rcv; do
  curl -O "https://pub-f0ad0e0dac0345408dcc95bda20beb42.r2.dev/datasets/parquet/00-latest/${section}.parquet"
done

import urllib.request

BASE = "https://pub-f0ad0e0dac0345408dcc95bda20beb42.r2.dev"

# Download latest monthly full bundle
urllib.request.urlretrieve(
    f"{BASE}/datasets/clinvar-gkm_00-latest.json.gz",
    "clinvar-gkm_00-latest.json.gz"
)

# Download a specific monthly full bundle
urllib.request.urlretrieve(
    f"{BASE}/datasets/clinvar-gkm_2026-06.json.gz",
    "clinvar-gkm_2026-06.json.gz"
)

# Download the latest weekly delta + manifest
urllib.request.urlretrieve(
    f"{BASE}/deltas/00-latest/clinvar-gkm-delta_00-latest.json.gz",
    "clinvar-gkm-delta_00-latest.json.gz"
)
urllib.request.urlretrieve(
    f"{BASE}/deltas/00-latest/manifest.json",
    "manifest.json"
)

# Download a specific weekly delta (release 2026-07-06 -> dir 2026-0706)
urllib.request.urlretrieve(
    f"{BASE}/deltas/2026-0706/clinvar-gkm-delta_2026-0706.json.gz",
    "clinvar-gkm-delta_2026-0706.json.gz"
)

# Download an archived full bundle from a prior year
urllib.request.urlretrieve(
    f"{BASE}/archives/2025/clinvar-gkm_2025-03.json.gz",
    "clinvar-gkm_2025-03.json.gz"
)

# Download a Parquet section from the latest monthly full
urllib.request.urlretrieve(
    f"{BASE}/datasets/parquet/00-latest/scv.parquet",
    "scv.parquet"
)

# Download a Parquet section from a specific monthly full (checkpoint-addressable)
urllib.request.urlretrieve(
    f"{BASE}/datasets/parquet/2026-06/scv.parquet",
    "scv.parquet"
)

deltas/2026-0706/
  clinvar-gkm-delta_2026-0706.json.gz   added + updated records (same section structure as the full bundle)
  manifest.json                         per-section adds, updates, and deletes for this release
  parquet/<section>.parquet             typed Parquet for the changed records only

{
  "release": "2026-07-06",
  "baseline_release": "2026-06-29",
  "compare_release": "2026-07-06",
  "pipeline_version": "clinvar-gkm vX.Y.Z @ 2026-07-06T00:00:00Z",
  "checkpoint_full": { "path": "datasets/clinvar-gkm_2026-06.json.gz", "release": "2026-06" },
  "sections": {
    "allele":  { "added": 812,  "updated": 34,  "deleted": ["ga4gh:VA.oldDigest1"] },
    "scv":     { "added": 1203, "updated": 517, "deleted": ["clinvar.submission:SCV000000001.1"] },
    "vcv":     { "added": 44,   "updated": 96,  "deleted": [] }
  },
  "counts": { "A": 2063, "U": 647, "D": 2 }
}

import gzip
import json
import urllib.request

BASE = "https://pub-f0ad0e0dac0345408dcc95bda20beb42.r2.dev"


def fetch_json(url):
    with urllib.request.urlopen(url) as r:
        return json.load(r)


def fetch_json_gz(url):
    with urllib.request.urlopen(url) as r:
        return json.loads(gzip.decompress(r.read()))


# 1. The latest delta's manifest names the monthly full to bootstrap from.
latest = fetch_json(f"{BASE}/deltas/00-latest/manifest.json")
checkpoint = latest["checkpoint_full"]        # {"path": "datasets/clinvar-gkm_2026-06.json.gz",
                                              #  "release": "2026-06"}  (None on initial rollout,
                                              #  before any monthly full exists)

# 2. Load that monthly full as the baseline state: {section: {key: record}}.
state = fetch_json_gz(f"{BASE}/{checkpoint['path']}")
checkpoint_month = checkpoint["release"]      # "2026-06"

# 3. From the index, take the dated weekly deltas published AFTER the checkpoint month,
#    oldest -> newest. (A delta dir is named YYYY-MMDD packed as "2026-0706", so the first
#    7 chars are its YYYY-MM.) Skip the "latest" mirror and any delta in the checkpoint
#    month or earlier — those are already folded into the monthly full.
index = fetch_json(f"{BASE}/index.json")
deltas = sorted(
    (d for d in index["deltas"]
     if d["release"] != "latest" and d["release"][:7] > checkpoint_month),
    key=lambda d: d["release"],
)

# 4. Replay each delta onto the baseline, verifying the chain as we go.
prev_compare = None
for d in deltas:
    manifest = fetch_json(f"{BASE}/{d['manifest']}")

    # Chain check: after the first, each delta must build on the previous compare_release.
    # The first delta builds on the checkpoint month's final release. A mismatch => a
    # missing week; re-bootstrap from the monthly full.
    if prev_compare is not None and manifest["baseline_release"] != prev_compare:
        raise SystemExit(
            f"broken chain before {manifest['release']}: re-bootstrap from a monthly full"
        )

    dirname = d["path"].strip("/").split("/")[-1]            # "2026-0706"
    delta = fetch_json_gz(f"{BASE}/{d['path']}clinvar-gkm-delta_{dirname}.json.gz")

    # 4a. Apply deletes (manifest only), then 4b. upsert added + updated records.
    for section, info in manifest["sections"].items():
        target = state.setdefault(section, {})
        for pk in info["deleted"]:
            target.pop(pk, None)
    for section, records in delta.items():
        state.setdefault(section, {}).update(records)

    prev_compare = manifest["compare_release"]

# `state` now reflects the most recent weekly release.

datasets/
  clinvar-gkm_00-latest.json.gz              latest monthly full bundle (stable URL)
  clinvar-gkm_YYYY-MM.json.gz                monthly full bundles (current year)

datasets/parquet/00-latest/
  {section}.parquet                          typed Parquet for the latest monthly full (stable URL)

datasets/parquet/YYYY-MM/
  {section}.parquet                          typed Parquet for a specific monthly full (current year)

deltas/00-latest/
  clinvar-gkm-delta_00-latest.json.gz        latest weekly delta bundle (stable URL)
  manifest.json                              latest delta manifest
  parquet/{section}.parquet                  typed Parquet for the latest delta

deltas/YYYY-MMDD/
  clinvar-gkm-delta_YYYY-MMDD.json.gz        weekly delta bundle (added + updated records)
  manifest.json                              per-release change manifest
  parquet/{section}.parquet                  typed Parquet for the changed records

archives/{YYYY}/
  clinvar-gkm_YYYY-MM.json.gz                monthly full bundles from prior years
  parquet/YYYY-MM/{section}.parquet          typed Parquet month sets from prior years

index.json                                   release index (datasets, archives, deltas)

# 00-latest = the newest monthly full; swap for a dated month (e.g. .../parquet/2026-06) to pin a release.
BASE="https://pub-f0ad0e0dac0345408dcc95bda20beb42.r2.dev/datasets/parquet/00-latest"
mkdir -p clinvar-gkm-parquet && cd clinvar-gkm-parquet

# Download specific sections
curl -O "${BASE}/scv.parquet"
curl -O "${BASE}/varcond-proposition.parquet"
curl -O "${BASE}/condition.parquet"

# Or download all 20 sections
for section in sequenceReference location allele copyNumberCount copyNumberChange \
               gene variation condition conditionSet therapy therapyGroup submitter \
               varcond-proposition vartumor-proposition vartherapy-proposition varcustom-proposition \
               evidenceLine vcv_evidenceLine rcv_evidenceLine \
               scv vcv rcv; do
  curl -O "${BASE}/${section}.parquet"
done

# Install DuckDB
brew install duckdb   # macOS
# or: pip install duckdb

# Query SCV statements
duckdb -c "
  SELECT id, classification.name AS classification, direction, strength.name AS strength, quality.name AS quality
  FROM 'scv.parquet'
  WHERE classification.name = 'Pathogenic'
  LIMIT 10;
"

# Count classifications across all SCVs
duckdb -c "
  SELECT classification.name AS classification, direction, COUNT(*) as n
  FROM 'scv.parquet'
  GROUP BY classification.name, direction
  ORDER BY n DESC;
"

# Join SCVs with propositions to find pathogenic variants for a specific condition
duckdb -c "
  SELECT s.id, s.classification.name AS classification, p.predicate, p.object_condition_id
  FROM 'scv.parquet' s
  JOIN 'varcond-proposition.parquet' p ON s.proposition_id = p.id
  WHERE s.classification.name = 'Pathogenic'
    AND p.object_condition_id LIKE '%clinvar.trait:9580%'
  LIMIT 10;
"

# Inspect the schema of any section
duckdb -c "DESCRIBE SELECT * FROM 'scv.parquet';"

import duckdb

df = duckdb.sql("""
    SELECT id, classification.name AS classification, direction, strength.name AS strength, quality.name AS quality
    FROM 'scv.parquet'
    WHERE classification.name = 'Pathogenic'
    LIMIT 100
""").df()

print(df)

import pandas as pd

# Load a section into a DataFrame
scv = pd.read_parquet("scv.parquet")

# Filter pathogenic SCVs — classification is a MappableConcept struct (dict), so read .name
pathogenic = scv[scv["classification"].map(lambda c: c and c.get("name")) == "Pathogenic"]
print(f"{len(pathogenic)} pathogenic SCVs")

# Access the full JSON when you need nested fields
import json
record = json.loads(pathogenic.iloc[0]["data"])
print(record["proposition"])

-- SCVs for BRCA1: classification, review status, condition
SELECT
    s.id AS scv_id,
    s.classification.name AS classification,
    s.direction,
    s.strength.name AS strength,
    s.quality.name AS review_status,
    p.gene_context_name AS gene,
    c.name AS condition_name,
    c.primaryCoding.code AS condition_code
FROM 'scv.parquet' s
JOIN 'varcond-proposition.parquet' p ON s.proposition_id = p.id
LEFT JOIN 'condition.parquet' c ON p.object_condition_id = c.id
WHERE p.gene_context_name = 'BRCA1'
ORDER BY s.classification.name;

-- Count SCVs by classification and review status for BRCA2
SELECT
    p.gene_context_name AS gene,
    s.classification.name AS classification,
    s.quality.name AS review_status,
    s.direction,
    COUNT(*) AS scv_count
FROM 'scv.parquet' s
JOIN 'varcond-proposition.parquet' p ON s.proposition_id = p.id
WHERE p.gene_context_name = 'BRCA2'
GROUP BY ALL
ORDER BY scv_count DESC;

-- Only expert panel and criteria-provided SCVs for a gene
SELECT
    s.id AS scv_id,
    s.classification.name AS classification,
    s.quality.name AS review_status,
    c.name AS condition_name
FROM 'scv.parquet' s
JOIN 'varcond-proposition.parquet' p ON s.proposition_id = p.id
LEFT JOIN 'condition.parquet' c ON p.object_condition_id = c.id
WHERE p.gene_context_name = 'TP53'
  AND s.quality.name IN ('criteria provided', 'reviewed by expert panel')
ORDER BY s.classification.name;

-- Compare pathogenicity classification distributions across genes
SELECT
    p.gene_context_name AS gene,
    s.classification.name AS classification,
    COUNT(*) AS n
FROM 'scv.parquet' s
JOIN 'varcond-proposition.parquet' p ON s.proposition_id = p.id
WHERE p.gene_context_name IN ('BRCA1', 'BRCA2', 'TP53', 'MLH1')
  AND s.quality.name = 'criteria provided'
GROUP BY gene, s.classification.name
ORDER BY gene, n DESC;

-- SCVs with submitter name and assertion method (from JSON)
SELECT
    s.id AS scv_id,
    s.classification.name AS classification,
    p.gene_context_name AS gene,
    json_extract_string(s.data, '$.contributions[0].agent.name') AS submitter,
    json_extract_string(s.data, '$.specifiedBy.name') AS method
FROM 'scv.parquet' s
JOIN 'varcond-proposition.parquet' p ON s.proposition_id = p.id
WHERE p.gene_context_name = 'BRCA1'
  AND s.classification.name = 'Pathogenic'
  AND s.quality.name = 'reviewed by expert panel'
LIMIT 20;

BASE="https://pub-f0ad0e0dac0345408dcc95bda20beb42.r2.dev"

# 1. The checkpoint monthly full named by the delta's manifest (e.g. 2026-06).
curl -s "${BASE}/deltas/00-latest/manifest.json" -o manifest.json
CKPT=$(python3 -c "import json;print(json.load(open('manifest.json'))['checkpoint_full']['release'])")

# 2. The checkpoint full Parquet set -> full/, and the delta set -> delta/.
mkdir -p full delta
for section in sequenceReference location allele copyNumberCount copyNumberChange \
               gene variation condition conditionSet therapy therapyGroup submitter \
               varcond-proposition vartumor-proposition vartherapy-proposition varcustom-proposition \
               evidenceLine vcv_evidenceLine rcv_evidenceLine scv vcv rcv; do
  curl -sf "${BASE}/datasets/parquet/${CKPT}/${section}.parquet" -o "full/${section}.parquet"  || true
  curl -sf "${BASE}/deltas/00-latest/parquet/${section}.parquet" -o "delta/${section}.parquet" || true
done

import duckdb, json, os, shutil

con = duckdb.connect()
manifest = json.load(open("manifest.json"))

# Start from a copy of the checkpoint full set; overwrite only the changed sections.
shutil.copytree("full", "updated", dirs_exist_ok=True)

for section, info in manifest["sections"].items():
    full    = f"full/{section}.parquet"      # checkpoint (baseline) section
    delta   = f"delta/{section}.parquet"     # added + updated rows (may be absent)
    out     = f"updated/{section}.parquet"
    deleted = info["deleted"]                # ids removed this release
    has_delta = os.path.exists(delta)

    if not os.path.exists(full):
        # Section new since the checkpoint: the delta rows ARE the section.
        if has_delta:
            shutil.copyfile(delta, out)
        continue

    if not has_delta:
        # Deletes only — filter the baseline, nothing to append.
        con.execute("""
            COPY (SELECT * FROM read_parquet(?) WHERE id NOT IN (SELECT unnest(?)))
            TO ? (FORMAT PARQUET)
        """, [full, deleted, out])
    else:
        # Drop updated + deleted keys from the baseline, then append the delta rows.
        con.execute("""
            COPY (
                SELECT * FROM read_parquet(?)                        -- baseline full
                WHERE id NOT IN (SELECT id FROM read_parquet(?))     -- drop updated keys
                  AND id NOT IN (SELECT unnest(?))                   -- drop deleted keys
                UNION ALL BY NAME
                SELECT * FROM read_parquet(?)                        -- append added + updated
            ) TO ? (FORMAT PARQUET)
        """, [full, delta, deleted, delta, out])

# `updated/<section>.parquet` now reflects the release the manifest names.