Upload analysis/camera_map_skill/gallery_lib.py with huggingface_hub
Browse files
analysis/camera_map_skill/gallery_lib.py
ADDED
|
@@ -0,0 +1,294 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Camera-map gallery for ImageNet-1K-Camera.
|
| 2 |
+
|
| 3 |
+
For N diverse samples: take the predicted (roll,pitch,vfov,k1) from
|
| 4 |
+
KangLiao/ImageNet-1K-Camera (val split), fetch the matching source image from
|
| 5 |
+
ILSVRC/imagenet-1k (val parquets), compute the up-field & latitude-field
|
| 6 |
+
(perspective fields, same as scripts/camera/cam_dataset_debug.py) and render
|
| 7 |
+
them overlaid on the image, side by side. All samples are tiled into one
|
| 8 |
+
gallery PNG.
|
| 9 |
+
"""
|
| 10 |
+
import argparse
|
| 11 |
+
import io
|
| 12 |
+
import json
|
| 13 |
+
import os
|
| 14 |
+
import re
|
| 15 |
+
import tarfile
|
| 16 |
+
|
| 17 |
+
import numpy as np
|
| 18 |
+
import torch
|
| 19 |
+
from PIL import Image, ImageDraw, ImageFont
|
| 20 |
+
|
| 21 |
+
import matplotlib
|
| 22 |
+
matplotlib.use("Agg")
|
| 23 |
+
import matplotlib.pyplot as plt
|
| 24 |
+
|
| 25 |
+
from huggingface_hub import hf_hub_download
|
| 26 |
+
import pyarrow.parquet as pq
|
| 27 |
+
|
| 28 |
+
from scripts.camera.geometry.camera import SimpleRadial
|
| 29 |
+
from scripts.camera.geometry.gravity import Gravity
|
| 30 |
+
from scripts.camera.geometry.perspective_fields import get_perspective_field
|
| 31 |
+
from scripts.camera.utils.conversions import fov2focal
|
| 32 |
+
from scripts.camera.visualization.viz2d import plot_vector_fields, plot_latitudes
|
| 33 |
+
|
| 34 |
+
CACHE = "/tmp/gallery"
|
| 35 |
+
DEG = 180.0 / np.pi
|
| 36 |
+
DATASET = os.environ.get("GALLERY_DATASET", "imagenet")
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
# ---- ImageNet (val split; key = ILSVRC val number) --------------------------
|
| 40 |
+
def _cap_imagenet():
|
| 41 |
+
p = hf_hub_download("KangLiao/ImageNet-1K-Camera", "val.tar", repo_type="dataset",
|
| 42 |
+
local_dir=os.path.join(CACHE, "in_cap"))
|
| 43 |
+
caps = {}
|
| 44 |
+
with tarfile.open(p) as t:
|
| 45 |
+
for m in t.getmembers():
|
| 46 |
+
if m.name.endswith(".json"):
|
| 47 |
+
d = json.loads(t.extractfile(m).read())
|
| 48 |
+
num = re.search(r"val_(\d+)", m.name)
|
| 49 |
+
if num and d.get("parse_ok", False):
|
| 50 |
+
caps[int(num.group(1))] = d
|
| 51 |
+
return caps
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def _src_imagenet(n_parquets):
|
| 55 |
+
imgs = {}
|
| 56 |
+
for i in range(n_parquets):
|
| 57 |
+
p = hf_hub_download("ILSVRC/imagenet-1k", f"data/validation-{i:05d}-of-00014.parquet",
|
| 58 |
+
repo_type="dataset", local_dir=os.path.join(CACHE, "in_src"))
|
| 59 |
+
pf = pq.ParquetFile(p)
|
| 60 |
+
for rg in range(pf.num_row_groups):
|
| 61 |
+
for img in pf.read_row_group(rg, columns=["image"]).to_pydict()["image"]:
|
| 62 |
+
num = re.search(r"val_(\d+)", img["path"])
|
| 63 |
+
if num and img.get("bytes"):
|
| 64 |
+
imgs[int(num.group(1))] = img["bytes"]
|
| 65 |
+
return imgs
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
# ---- COCO (val split; key = COCO image_id) ----------------------------------
|
| 69 |
+
def _cap_coco():
|
| 70 |
+
from huggingface_hub import HfApi
|
| 71 |
+
caps = {}
|
| 72 |
+
for f in [x for x in HfApi().list_repo_files("KangLiao/COCO-Camera", repo_type="dataset")
|
| 73 |
+
if "coco_val_" in x and x.endswith(".tar")]:
|
| 74 |
+
p = hf_hub_download("KangLiao/COCO-Camera", f, repo_type="dataset",
|
| 75 |
+
local_dir=os.path.join(CACHE, "coco_cap"))
|
| 76 |
+
with tarfile.open(p) as t:
|
| 77 |
+
for m in t.getmembers():
|
| 78 |
+
if m.name.endswith(".json"):
|
| 79 |
+
d = json.loads(t.extractfile(m).read())
|
| 80 |
+
if d.get("parse_ok", False):
|
| 81 |
+
caps[int(os.path.basename(m.name)[:-5])] = d
|
| 82 |
+
return caps
|
| 83 |
+
|
| 84 |
+
|
| 85 |
+
def _src_coco(n_parquets):
|
| 86 |
+
from huggingface_hub import HfApi
|
| 87 |
+
vals = sorted(x for x in HfApi().list_repo_files("detection-datasets/coco", repo_type="dataset")
|
| 88 |
+
if os.path.basename(x).startswith("val-") and x.endswith(".parquet"))
|
| 89 |
+
imgs = {}
|
| 90 |
+
for f in vals[:max(n_parquets, 2)]:
|
| 91 |
+
p = hf_hub_download("detection-datasets/coco", f, repo_type="dataset",
|
| 92 |
+
local_dir=os.path.join(CACHE, "coco_src"))
|
| 93 |
+
pf = pq.ParquetFile(p)
|
| 94 |
+
for rg in range(pf.num_row_groups):
|
| 95 |
+
t = pf.read_row_group(rg, columns=["image_id", "image"]).to_pydict()
|
| 96 |
+
for iid, img in zip(t["image_id"], t["image"]):
|
| 97 |
+
if img.get("bytes"):
|
| 98 |
+
imgs[int(iid)] = img["bytes"]
|
| 99 |
+
return imgs
|
| 100 |
+
|
| 101 |
+
|
| 102 |
+
# ---- CC12M (key = "<shard>/<index>"; source = pixparse/cc12m-wds) -----------
|
| 103 |
+
CC12M_SHARDS = list(range(6)) # which shards to sample from (limits download)
|
| 104 |
+
|
| 105 |
+
|
| 106 |
+
def _cap_cc12m():
|
| 107 |
+
caps = {}
|
| 108 |
+
for s in CC12M_SHARDS:
|
| 109 |
+
p = hf_hub_download("KangLiao/CC12M-Camera", f"{s:04d}.tar", repo_type="dataset",
|
| 110 |
+
local_dir=os.path.join(CACHE, "cc12m_cap"))
|
| 111 |
+
with tarfile.open(p) as t:
|
| 112 |
+
for m in t.getmembers():
|
| 113 |
+
if m.name.endswith(".json"):
|
| 114 |
+
d = json.loads(t.extractfile(m).read())
|
| 115 |
+
if d.get("parse_ok", False):
|
| 116 |
+
caps[f"{s:04d}/{os.path.basename(m.name)[:-5]}"] = d
|
| 117 |
+
return caps
|
| 118 |
+
|
| 119 |
+
|
| 120 |
+
def _src_cc12m(_n):
|
| 121 |
+
imgs = {}
|
| 122 |
+
for s in CC12M_SHARDS:
|
| 123 |
+
p = hf_hub_download("pixparse/cc12m-wds", f"cc12m-train-{s:04d}.tar", repo_type="dataset",
|
| 124 |
+
local_dir=os.path.join(CACHE, "cc12m_src"))
|
| 125 |
+
with tarfile.open(p) as t:
|
| 126 |
+
for m in t.getmembers():
|
| 127 |
+
if m.name.endswith(".jpg"):
|
| 128 |
+
imgs[f"{s:04d}/{os.path.basename(m.name)[:-4]}"] = t.extractfile(m).read()
|
| 129 |
+
return imgs
|
| 130 |
+
|
| 131 |
+
|
| 132 |
+
# ---- Megalith (caption carries `url`; source fetched from that url) ---------
|
| 133 |
+
MEGALITH_SHARDS = list(range(6))
|
| 134 |
+
|
| 135 |
+
|
| 136 |
+
def _cap_megalith():
|
| 137 |
+
caps = {}
|
| 138 |
+
for s in MEGALITH_SHARDS:
|
| 139 |
+
p = hf_hub_download("KangLiao/Megalith-10M-Camera", f"{s:05d}.tar", repo_type="dataset",
|
| 140 |
+
local_dir=os.path.join(CACHE, "mega_cap"))
|
| 141 |
+
with tarfile.open(p) as t:
|
| 142 |
+
for m in t.getmembers():
|
| 143 |
+
if m.name.endswith(".json"):
|
| 144 |
+
d = json.loads(t.extractfile(m).read())
|
| 145 |
+
if d.get("parse_ok", False) and d.get("url"):
|
| 146 |
+
caps[f"{s:05d}/{os.path.basename(m.name)[:-5]}"] = d
|
| 147 |
+
return caps
|
| 148 |
+
|
| 149 |
+
|
| 150 |
+
def fetch_url(url, timeout=12):
|
| 151 |
+
"""Download bytes from a URL, bypassing the (flaky) env proxy. None on fail."""
|
| 152 |
+
import requests
|
| 153 |
+
try:
|
| 154 |
+
r = requests.get(url, timeout=timeout, proxies={"http": None, "https": None})
|
| 155 |
+
r.raise_for_status()
|
| 156 |
+
return r.content
|
| 157 |
+
except Exception:
|
| 158 |
+
return None
|
| 159 |
+
|
| 160 |
+
|
| 161 |
+
def load_captions():
|
| 162 |
+
return {"coco": _cap_coco, "cc12m": _cap_cc12m,
|
| 163 |
+
"megalith": _cap_megalith}.get(DATASET, _cap_imagenet)()
|
| 164 |
+
|
| 165 |
+
|
| 166 |
+
def load_source_images(n_parquets):
|
| 167 |
+
# megalith fetches per-selected-sample via fetch_url (handled by callers)
|
| 168 |
+
return {"coco": _src_coco, "cc12m": _src_cc12m}.get(DATASET, _src_imagenet)(n_parquets)
|
| 169 |
+
|
| 170 |
+
|
| 171 |
+
def select_diverse(candidates, n):
|
| 172 |
+
"""Farthest-point sampling in (roll,pitch,vfov) degree space -> diversity,
|
| 173 |
+
guaranteed to include non-trivial (non-zero) roll/pitch/fov extremes."""
|
| 174 |
+
keys = list(candidates)
|
| 175 |
+
X = np.array([[candidates[k]["roll"] * DEG, candidates[k]["pitch"] * DEG,
|
| 176 |
+
candidates[k]["vfov"] * DEG] for k in keys], dtype=np.float64)
|
| 177 |
+
Xn = (X - X.mean(0)) / (X.std(0) + 1e-6)
|
| 178 |
+
# seed: the most extreme sample (largest deviation from mean)
|
| 179 |
+
seed = int(np.argmax((Xn ** 2).sum(1)))
|
| 180 |
+
chosen = [seed]
|
| 181 |
+
d = np.linalg.norm(Xn - Xn[seed], axis=1)
|
| 182 |
+
while len(chosen) < min(n, len(keys)):
|
| 183 |
+
nxt = int(np.argmax(d))
|
| 184 |
+
chosen.append(nxt)
|
| 185 |
+
d = np.minimum(d, np.linalg.norm(Xn - Xn[nxt], axis=1))
|
| 186 |
+
return [keys[i] for i in chosen]
|
| 187 |
+
|
| 188 |
+
|
| 189 |
+
def prep_image(pil, size=640):
|
| 190 |
+
w, h = pil.size
|
| 191 |
+
if w >= h:
|
| 192 |
+
nw, nh = size, max(1, round(h * size / w))
|
| 193 |
+
else:
|
| 194 |
+
nh, nw = size, max(1, round(w * size / h))
|
| 195 |
+
pil = pil.resize((nw, nh))
|
| 196 |
+
canvas = Image.new("RGB", (size, size), (0, 0, 0))
|
| 197 |
+
canvas.paste(pil, ((size - nw) // 2, (size - nh) // 2))
|
| 198 |
+
return canvas
|
| 199 |
+
|
| 200 |
+
|
| 201 |
+
def compute_fields(roll, pitch, vfov, k1, h=640, w=640):
|
| 202 |
+
f = fov2focal(torch.tensor(float(vfov)), h)
|
| 203 |
+
params = torch.tensor([w, h, float(f), float(f), w / 2, h / 2, float(k1), 0]).float()
|
| 204 |
+
cam = SimpleRadial(params).float()
|
| 205 |
+
grav = Gravity.from_rp(torch.tensor(float(roll)), torch.tensor(float(pitch)))
|
| 206 |
+
up, lat = get_perspective_field(cam, grav, use_up=True, use_latitude=True)
|
| 207 |
+
return up[0], lat[0] # up (2,H,W), lat (1,H,W)
|
| 208 |
+
|
| 209 |
+
|
| 210 |
+
def render_pair(img640, up_field, lat_field, panel_px=380):
|
| 211 |
+
"""One sample -> a [up | lat] RGB image (numpy)."""
|
| 212 |
+
imnp = np.asarray(img640).astype(np.float32) / 255.0
|
| 213 |
+
fig, axes = plt.subplots(1, 2, figsize=(2 * panel_px / 100, panel_px / 100), dpi=100)
|
| 214 |
+
for ax in axes:
|
| 215 |
+
ax.imshow(imnp)
|
| 216 |
+
ax.set_axis_off()
|
| 217 |
+
ax.set_xlim([0, 640]); ax.set_ylim([640, 0])
|
| 218 |
+
plot_vector_fields([up_field], axes=[axes[0]])
|
| 219 |
+
lat_deg = (lat_field[0] * DEG)
|
| 220 |
+
plot_latitudes([lat_deg], is_radians=False, axes=[axes[1]])
|
| 221 |
+
axes[0].set_title("up field", fontsize=11)
|
| 222 |
+
axes[1].set_title("latitude field", fontsize=11)
|
| 223 |
+
fig.subplots_adjust(left=0, right=1, top=0.93, bottom=0, wspace=0.02)
|
| 224 |
+
fig.canvas.draw()
|
| 225 |
+
buf = np.asarray(fig.canvas.buffer_rgba())[..., :3].copy()
|
| 226 |
+
plt.close(fig)
|
| 227 |
+
return buf
|
| 228 |
+
|
| 229 |
+
|
| 230 |
+
def build_gallery(panels, params, ncols, out_path):
|
| 231 |
+
ph, pw = panels[0].shape[:2]
|
| 232 |
+
gap, top = 16, 90
|
| 233 |
+
cap_h = 30
|
| 234 |
+
cell_h = ph + cap_h
|
| 235 |
+
nrows = (len(panels) + ncols - 1) // ncols
|
| 236 |
+
W = ncols * pw + (ncols + 1) * gap
|
| 237 |
+
H = top + nrows * (cell_h + gap) + gap
|
| 238 |
+
canvas = Image.new("RGB", (W, H), (245, 246, 248))
|
| 239 |
+
draw = ImageDraw.Draw(canvas)
|
| 240 |
+
try:
|
| 241 |
+
ft = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 40)
|
| 242 |
+
fs = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 16)
|
| 243 |
+
except Exception:
|
| 244 |
+
ft = ImageFont.load_default(); fs = ImageFont.load_default()
|
| 245 |
+
draw.text((gap, 26), "ImageNet-1K-Camera — Camera-Map Gallery (up & latitude fields)",
|
| 246 |
+
fill=(30, 30, 40), font=ft)
|
| 247 |
+
for i, (pan, pr) in enumerate(zip(panels, params)):
|
| 248 |
+
r, c = divmod(i, ncols)
|
| 249 |
+
x = gap + c * (pw + gap)
|
| 250 |
+
y = top + r * (cell_h + gap)
|
| 251 |
+
canvas.paste(Image.fromarray(pan), (x, y))
|
| 252 |
+
txt = (f"roll {pr['roll']*DEG:+.1f}° pitch {pr['pitch']*DEG:+.1f}° "
|
| 253 |
+
f"fov {pr['vfov']*DEG:.1f}°")
|
| 254 |
+
draw.text((x + 6, y + ph + 6), txt, fill=(60, 60, 70), font=fs)
|
| 255 |
+
canvas.save(out_path)
|
| 256 |
+
print("saved gallery ->", out_path, canvas.size)
|
| 257 |
+
|
| 258 |
+
|
| 259 |
+
def main():
|
| 260 |
+
ap = argparse.ArgumentParser()
|
| 261 |
+
ap.add_argument("--n", type=int, default=50)
|
| 262 |
+
ap.add_argument("--ncols", type=int, default=5)
|
| 263 |
+
ap.add_argument("--parquets", type=int, default=6)
|
| 264 |
+
ap.add_argument("--out", default="output/imagenet1k_camera_map_gallery.png")
|
| 265 |
+
args = ap.parse_args()
|
| 266 |
+
|
| 267 |
+
print("loading captions ..."); caps = load_captions()
|
| 268 |
+
print("loading source images ..."); imgs = load_source_images(args.parquets)
|
| 269 |
+
cand = {k: caps[k] for k in caps if k in imgs}
|
| 270 |
+
print(f"candidates with both caption+image: {len(cand)}")
|
| 271 |
+
picks = select_diverse(cand, args.n)
|
| 272 |
+
print(f"selected {len(picks)} diverse samples")
|
| 273 |
+
|
| 274 |
+
panels, params = [], []
|
| 275 |
+
for k in picks:
|
| 276 |
+
pr = cand[k]
|
| 277 |
+
pil = Image.open(io.BytesIO(imgs[k])).convert("RGB")
|
| 278 |
+
img640 = prep_image(pil)
|
| 279 |
+
up, lat = compute_fields(pr["roll"], pr["pitch"], pr["vfov"], pr["k1"])
|
| 280 |
+
panels.append(render_pair(img640, up, lat))
|
| 281 |
+
params.append(pr)
|
| 282 |
+
os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
|
| 283 |
+
build_gallery(panels, params, args.ncols, args.out)
|
| 284 |
+
# report the diversity actually achieved
|
| 285 |
+
rs = np.array([p["roll"] * DEG for p in params])
|
| 286 |
+
ps = np.array([p["pitch"] * DEG for p in params])
|
| 287 |
+
vs = np.array([p["vfov"] * DEG for p in params])
|
| 288 |
+
print(f"roll range [{rs.min():.1f},{rs.max():.1f}] std {rs.std():.1f}")
|
| 289 |
+
print(f"pitch range [{ps.min():.1f},{ps.max():.1f}] std {ps.std():.1f}")
|
| 290 |
+
print(f"fov range [{vs.min():.1f},{vs.max():.1f}] std {vs.std():.1f}")
|
| 291 |
+
|
| 292 |
+
|
| 293 |
+
if __name__ == "__main__":
|
| 294 |
+
main()
|