KangLiao commited on
Commit
a9a98e3
·
verified ·
1 Parent(s): 942a99a

Upload analysis/camera_map_skill/gallery_lib.py with huggingface_hub

Browse files
analysis/camera_map_skill/gallery_lib.py ADDED
@@ -0,0 +1,294 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ """Camera-map gallery for ImageNet-1K-Camera.
2
+
3
+ For N diverse samples: take the predicted (roll,pitch,vfov,k1) from
4
+ KangLiao/ImageNet-1K-Camera (val split), fetch the matching source image from
5
+ ILSVRC/imagenet-1k (val parquets), compute the up-field & latitude-field
6
+ (perspective fields, same as scripts/camera/cam_dataset_debug.py) and render
7
+ them overlaid on the image, side by side. All samples are tiled into one
8
+ gallery PNG.
9
+ """
10
+ import argparse
11
+ import io
12
+ import json
13
+ import os
14
+ import re
15
+ import tarfile
16
+
17
+ import numpy as np
18
+ import torch
19
+ from PIL import Image, ImageDraw, ImageFont
20
+
21
+ import matplotlib
22
+ matplotlib.use("Agg")
23
+ import matplotlib.pyplot as plt
24
+
25
+ from huggingface_hub import hf_hub_download
26
+ import pyarrow.parquet as pq
27
+
28
+ from scripts.camera.geometry.camera import SimpleRadial
29
+ from scripts.camera.geometry.gravity import Gravity
30
+ from scripts.camera.geometry.perspective_fields import get_perspective_field
31
+ from scripts.camera.utils.conversions import fov2focal
32
+ from scripts.camera.visualization.viz2d import plot_vector_fields, plot_latitudes
33
+
34
+ CACHE = "/tmp/gallery"
35
+ DEG = 180.0 / np.pi
36
+ DATASET = os.environ.get("GALLERY_DATASET", "imagenet")
37
+
38
+
39
+ # ---- ImageNet (val split; key = ILSVRC val number) --------------------------
40
+ def _cap_imagenet():
41
+ p = hf_hub_download("KangLiao/ImageNet-1K-Camera", "val.tar", repo_type="dataset",
42
+ local_dir=os.path.join(CACHE, "in_cap"))
43
+ caps = {}
44
+ with tarfile.open(p) as t:
45
+ for m in t.getmembers():
46
+ if m.name.endswith(".json"):
47
+ d = json.loads(t.extractfile(m).read())
48
+ num = re.search(r"val_(\d+)", m.name)
49
+ if num and d.get("parse_ok", False):
50
+ caps[int(num.group(1))] = d
51
+ return caps
52
+
53
+
54
+ def _src_imagenet(n_parquets):
55
+ imgs = {}
56
+ for i in range(n_parquets):
57
+ p = hf_hub_download("ILSVRC/imagenet-1k", f"data/validation-{i:05d}-of-00014.parquet",
58
+ repo_type="dataset", local_dir=os.path.join(CACHE, "in_src"))
59
+ pf = pq.ParquetFile(p)
60
+ for rg in range(pf.num_row_groups):
61
+ for img in pf.read_row_group(rg, columns=["image"]).to_pydict()["image"]:
62
+ num = re.search(r"val_(\d+)", img["path"])
63
+ if num and img.get("bytes"):
64
+ imgs[int(num.group(1))] = img["bytes"]
65
+ return imgs
66
+
67
+
68
+ # ---- COCO (val split; key = COCO image_id) ----------------------------------
69
+ def _cap_coco():
70
+ from huggingface_hub import HfApi
71
+ caps = {}
72
+ for f in [x for x in HfApi().list_repo_files("KangLiao/COCO-Camera", repo_type="dataset")
73
+ if "coco_val_" in x and x.endswith(".tar")]:
74
+ p = hf_hub_download("KangLiao/COCO-Camera", f, repo_type="dataset",
75
+ local_dir=os.path.join(CACHE, "coco_cap"))
76
+ with tarfile.open(p) as t:
77
+ for m in t.getmembers():
78
+ if m.name.endswith(".json"):
79
+ d = json.loads(t.extractfile(m).read())
80
+ if d.get("parse_ok", False):
81
+ caps[int(os.path.basename(m.name)[:-5])] = d
82
+ return caps
83
+
84
+
85
+ def _src_coco(n_parquets):
86
+ from huggingface_hub import HfApi
87
+ vals = sorted(x for x in HfApi().list_repo_files("detection-datasets/coco", repo_type="dataset")
88
+ if os.path.basename(x).startswith("val-") and x.endswith(".parquet"))
89
+ imgs = {}
90
+ for f in vals[:max(n_parquets, 2)]:
91
+ p = hf_hub_download("detection-datasets/coco", f, repo_type="dataset",
92
+ local_dir=os.path.join(CACHE, "coco_src"))
93
+ pf = pq.ParquetFile(p)
94
+ for rg in range(pf.num_row_groups):
95
+ t = pf.read_row_group(rg, columns=["image_id", "image"]).to_pydict()
96
+ for iid, img in zip(t["image_id"], t["image"]):
97
+ if img.get("bytes"):
98
+ imgs[int(iid)] = img["bytes"]
99
+ return imgs
100
+
101
+
102
+ # ---- CC12M (key = "<shard>/<index>"; source = pixparse/cc12m-wds) -----------
103
+ CC12M_SHARDS = list(range(6)) # which shards to sample from (limits download)
104
+
105
+
106
+ def _cap_cc12m():
107
+ caps = {}
108
+ for s in CC12M_SHARDS:
109
+ p = hf_hub_download("KangLiao/CC12M-Camera", f"{s:04d}.tar", repo_type="dataset",
110
+ local_dir=os.path.join(CACHE, "cc12m_cap"))
111
+ with tarfile.open(p) as t:
112
+ for m in t.getmembers():
113
+ if m.name.endswith(".json"):
114
+ d = json.loads(t.extractfile(m).read())
115
+ if d.get("parse_ok", False):
116
+ caps[f"{s:04d}/{os.path.basename(m.name)[:-5]}"] = d
117
+ return caps
118
+
119
+
120
+ def _src_cc12m(_n):
121
+ imgs = {}
122
+ for s in CC12M_SHARDS:
123
+ p = hf_hub_download("pixparse/cc12m-wds", f"cc12m-train-{s:04d}.tar", repo_type="dataset",
124
+ local_dir=os.path.join(CACHE, "cc12m_src"))
125
+ with tarfile.open(p) as t:
126
+ for m in t.getmembers():
127
+ if m.name.endswith(".jpg"):
128
+ imgs[f"{s:04d}/{os.path.basename(m.name)[:-4]}"] = t.extractfile(m).read()
129
+ return imgs
130
+
131
+
132
+ # ---- Megalith (caption carries `url`; source fetched from that url) ---------
133
+ MEGALITH_SHARDS = list(range(6))
134
+
135
+
136
+ def _cap_megalith():
137
+ caps = {}
138
+ for s in MEGALITH_SHARDS:
139
+ p = hf_hub_download("KangLiao/Megalith-10M-Camera", f"{s:05d}.tar", repo_type="dataset",
140
+ local_dir=os.path.join(CACHE, "mega_cap"))
141
+ with tarfile.open(p) as t:
142
+ for m in t.getmembers():
143
+ if m.name.endswith(".json"):
144
+ d = json.loads(t.extractfile(m).read())
145
+ if d.get("parse_ok", False) and d.get("url"):
146
+ caps[f"{s:05d}/{os.path.basename(m.name)[:-5]}"] = d
147
+ return caps
148
+
149
+
150
+ def fetch_url(url, timeout=12):
151
+ """Download bytes from a URL, bypassing the (flaky) env proxy. None on fail."""
152
+ import requests
153
+ try:
154
+ r = requests.get(url, timeout=timeout, proxies={"http": None, "https": None})
155
+ r.raise_for_status()
156
+ return r.content
157
+ except Exception:
158
+ return None
159
+
160
+
161
+ def load_captions():
162
+ return {"coco": _cap_coco, "cc12m": _cap_cc12m,
163
+ "megalith": _cap_megalith}.get(DATASET, _cap_imagenet)()
164
+
165
+
166
+ def load_source_images(n_parquets):
167
+ # megalith fetches per-selected-sample via fetch_url (handled by callers)
168
+ return {"coco": _src_coco, "cc12m": _src_cc12m}.get(DATASET, _src_imagenet)(n_parquets)
169
+
170
+
171
+ def select_diverse(candidates, n):
172
+ """Farthest-point sampling in (roll,pitch,vfov) degree space -> diversity,
173
+ guaranteed to include non-trivial (non-zero) roll/pitch/fov extremes."""
174
+ keys = list(candidates)
175
+ X = np.array([[candidates[k]["roll"] * DEG, candidates[k]["pitch"] * DEG,
176
+ candidates[k]["vfov"] * DEG] for k in keys], dtype=np.float64)
177
+ Xn = (X - X.mean(0)) / (X.std(0) + 1e-6)
178
+ # seed: the most extreme sample (largest deviation from mean)
179
+ seed = int(np.argmax((Xn ** 2).sum(1)))
180
+ chosen = [seed]
181
+ d = np.linalg.norm(Xn - Xn[seed], axis=1)
182
+ while len(chosen) < min(n, len(keys)):
183
+ nxt = int(np.argmax(d))
184
+ chosen.append(nxt)
185
+ d = np.minimum(d, np.linalg.norm(Xn - Xn[nxt], axis=1))
186
+ return [keys[i] for i in chosen]
187
+
188
+
189
+ def prep_image(pil, size=640):
190
+ w, h = pil.size
191
+ if w >= h:
192
+ nw, nh = size, max(1, round(h * size / w))
193
+ else:
194
+ nh, nw = size, max(1, round(w * size / h))
195
+ pil = pil.resize((nw, nh))
196
+ canvas = Image.new("RGB", (size, size), (0, 0, 0))
197
+ canvas.paste(pil, ((size - nw) // 2, (size - nh) // 2))
198
+ return canvas
199
+
200
+
201
+ def compute_fields(roll, pitch, vfov, k1, h=640, w=640):
202
+ f = fov2focal(torch.tensor(float(vfov)), h)
203
+ params = torch.tensor([w, h, float(f), float(f), w / 2, h / 2, float(k1), 0]).float()
204
+ cam = SimpleRadial(params).float()
205
+ grav = Gravity.from_rp(torch.tensor(float(roll)), torch.tensor(float(pitch)))
206
+ up, lat = get_perspective_field(cam, grav, use_up=True, use_latitude=True)
207
+ return up[0], lat[0] # up (2,H,W), lat (1,H,W)
208
+
209
+
210
+ def render_pair(img640, up_field, lat_field, panel_px=380):
211
+ """One sample -> a [up | lat] RGB image (numpy)."""
212
+ imnp = np.asarray(img640).astype(np.float32) / 255.0
213
+ fig, axes = plt.subplots(1, 2, figsize=(2 * panel_px / 100, panel_px / 100), dpi=100)
214
+ for ax in axes:
215
+ ax.imshow(imnp)
216
+ ax.set_axis_off()
217
+ ax.set_xlim([0, 640]); ax.set_ylim([640, 0])
218
+ plot_vector_fields([up_field], axes=[axes[0]])
219
+ lat_deg = (lat_field[0] * DEG)
220
+ plot_latitudes([lat_deg], is_radians=False, axes=[axes[1]])
221
+ axes[0].set_title("up field", fontsize=11)
222
+ axes[1].set_title("latitude field", fontsize=11)
223
+ fig.subplots_adjust(left=0, right=1, top=0.93, bottom=0, wspace=0.02)
224
+ fig.canvas.draw()
225
+ buf = np.asarray(fig.canvas.buffer_rgba())[..., :3].copy()
226
+ plt.close(fig)
227
+ return buf
228
+
229
+
230
+ def build_gallery(panels, params, ncols, out_path):
231
+ ph, pw = panels[0].shape[:2]
232
+ gap, top = 16, 90
233
+ cap_h = 30
234
+ cell_h = ph + cap_h
235
+ nrows = (len(panels) + ncols - 1) // ncols
236
+ W = ncols * pw + (ncols + 1) * gap
237
+ H = top + nrows * (cell_h + gap) + gap
238
+ canvas = Image.new("RGB", (W, H), (245, 246, 248))
239
+ draw = ImageDraw.Draw(canvas)
240
+ try:
241
+ ft = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans-Bold.ttf", 40)
242
+ fs = ImageFont.truetype("/usr/share/fonts/truetype/dejavu/DejaVuSans.ttf", 16)
243
+ except Exception:
244
+ ft = ImageFont.load_default(); fs = ImageFont.load_default()
245
+ draw.text((gap, 26), "ImageNet-1K-Camera — Camera-Map Gallery (up & latitude fields)",
246
+ fill=(30, 30, 40), font=ft)
247
+ for i, (pan, pr) in enumerate(zip(panels, params)):
248
+ r, c = divmod(i, ncols)
249
+ x = gap + c * (pw + gap)
250
+ y = top + r * (cell_h + gap)
251
+ canvas.paste(Image.fromarray(pan), (x, y))
252
+ txt = (f"roll {pr['roll']*DEG:+.1f}° pitch {pr['pitch']*DEG:+.1f}° "
253
+ f"fov {pr['vfov']*DEG:.1f}°")
254
+ draw.text((x + 6, y + ph + 6), txt, fill=(60, 60, 70), font=fs)
255
+ canvas.save(out_path)
256
+ print("saved gallery ->", out_path, canvas.size)
257
+
258
+
259
+ def main():
260
+ ap = argparse.ArgumentParser()
261
+ ap.add_argument("--n", type=int, default=50)
262
+ ap.add_argument("--ncols", type=int, default=5)
263
+ ap.add_argument("--parquets", type=int, default=6)
264
+ ap.add_argument("--out", default="output/imagenet1k_camera_map_gallery.png")
265
+ args = ap.parse_args()
266
+
267
+ print("loading captions ..."); caps = load_captions()
268
+ print("loading source images ..."); imgs = load_source_images(args.parquets)
269
+ cand = {k: caps[k] for k in caps if k in imgs}
270
+ print(f"candidates with both caption+image: {len(cand)}")
271
+ picks = select_diverse(cand, args.n)
272
+ print(f"selected {len(picks)} diverse samples")
273
+
274
+ panels, params = [], []
275
+ for k in picks:
276
+ pr = cand[k]
277
+ pil = Image.open(io.BytesIO(imgs[k])).convert("RGB")
278
+ img640 = prep_image(pil)
279
+ up, lat = compute_fields(pr["roll"], pr["pitch"], pr["vfov"], pr["k1"])
280
+ panels.append(render_pair(img640, up, lat))
281
+ params.append(pr)
282
+ os.makedirs(os.path.dirname(args.out) or ".", exist_ok=True)
283
+ build_gallery(panels, params, args.ncols, args.out)
284
+ # report the diversity actually achieved
285
+ rs = np.array([p["roll"] * DEG for p in params])
286
+ ps = np.array([p["pitch"] * DEG for p in params])
287
+ vs = np.array([p["vfov"] * DEG for p in params])
288
+ print(f"roll range [{rs.min():.1f},{rs.max():.1f}] std {rs.std():.1f}")
289
+ print(f"pitch range [{ps.min():.1f},{ps.max():.1f}] std {ps.std():.1f}")
290
+ print(f"fov range [{vs.min():.1f},{vs.max():.1f}] std {vs.std():.1f}")
291
+
292
+
293
+ if __name__ == "__main__":
294
+ main()