mirror of
https://github.com/allaunthefox/Research-Stack.git
synced 2026-07-31 03:05:21 +00:00
79 lines
2.6 KiB
Python
79 lines
2.6 KiB
Python
# ==============================================================================
|
|
# COPYRIGHT NO ONE EVERYWHERE LLC (WYOMING HOLDING COMPANY)
|
|
# PROJECT: SOVEREIGN STACK
|
|
# This artifact is entirely proprietary and cryptographically proven.
|
|
# Open-Source usage requires explicit permission from Brandon Scott Schneider.
|
|
# ==============================================================================
|
|
from pathlib import Path
|
|
import json
|
|
from datasets import load_dataset
|
|
from huggingface_hub import HfApi
|
|
import geopandas as gpd
|
|
|
|
OUTPUT_DIR = Path("data_baselines/geospatial").absolute()
|
|
OUTPUT_DIR.mkdir(parents=True, exist_ok=True)
|
|
LOG = Path("data_baselines/geospatial_ingest.log")
|
|
|
|
|
|
def normalize_crs(df, target_epsg=4326):
|
|
if hasattr(df, "to_crs"):
|
|
return df.to_crs(epsg=target_epsg)
|
|
return df
|
|
|
|
|
|
def convert_geo(table, out_path):
|
|
if hasattr(table, "to_pandas"):
|
|
df = table.to_pandas()
|
|
else:
|
|
df = table
|
|
|
|
if "geometry" in df.columns:
|
|
gdf = gpd.GeoDataFrame(df, geometry="geometry")
|
|
gdf = normalize_crs(gdf)
|
|
gdf.to_file(out_path, driver="GeoJSON")
|
|
return True
|
|
return False
|
|
|
|
|
|
def ingest_dataset(dataset_id):
|
|
print(f"[INGEST] {dataset_id}")
|
|
ds = load_dataset(dataset_id)
|
|
meta = []
|
|
for split, table in ds.items():
|
|
out_dir = OUTPUT_DIR / dataset_id.replace("/", "__")
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
safe_split = split.replace("/", "_")
|
|
target_parquet = out_dir / f"{safe_split}.parquet"
|
|
try:
|
|
table.to_parquet(target_parquet, index=False)
|
|
except Exception as e:
|
|
print(f" cannot parquet {dataset_id}/{split}: {e}")
|
|
continue
|
|
|
|
geojs = out_dir / f"{safe_split}.geojson"
|
|
try:
|
|
convert_geo(table, geojs)
|
|
except Exception as e:
|
|
print(f" geo conversion fail {dataset_id}/{split}: {e}")
|
|
|
|
meta.append({"dataset": dataset_id, "split": split, "path": str(target_parquet), "geojson": str(geojs) if geojs.exists() else None})
|
|
|
|
with LOG.open("a", encoding="utf-8") as f:
|
|
f.write(json.dumps({"dataset": dataset_id, "meta": meta}) + "\n")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
api = HfApi()
|
|
# geospatial + datasets library=datasets trending set
|
|
datasets = []
|
|
results = api.list_datasets(filter="geospatial", sort="trending_score", limit=200)
|
|
for r in results:
|
|
if r.id not in datasets:
|
|
datasets.append(r.id)
|
|
|
|
print(f"Found {len(datasets)} datasets")
|
|
for dsid in datasets:
|
|
try:
|
|
ingest_dataset(dsid)
|
|
except Exception as e:
|
|
print(f"FAILED {dsid}: {e}")
|