Spaces:
Sleeping
Sleeping
Commit
·
6fa035b
1
Parent(s):
90af011
Add code
Browse files- app.py +44 -0
- processing.py +77 -0
- requirements.txt +1 -0
app.py
ADDED
|
@@ -0,0 +1,44 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from processing import process_spaces
|
| 2 |
+
|
| 3 |
+
from apscheduler.schedulers.background import BackgroundScheduler
|
| 4 |
+
from huggingface_hub import hf_hub_url
|
| 5 |
+
from fastapi import FastAPI
|
| 6 |
+
from fastapi.responses import HTMLResponse
|
| 7 |
+
import uvicorn
|
| 8 |
+
import requests
|
| 9 |
+
|
| 10 |
+
def batch(iterable, n=1):
|
| 11 |
+
l = len(iterable)
|
| 12 |
+
for ndx in range(0, l, n):
|
| 13 |
+
yield iterable[ndx:min(ndx + n, l)]
|
| 14 |
+
|
| 15 |
+
|
| 16 |
+
def embed_theme_spaces():
|
| 17 |
+
|
| 18 |
+
subdomains = requests.get(hf_hub_url("freddyaboulton/gradio-theme-subdomains",
|
| 19 |
+
filename="subdomains.json",
|
| 20 |
+
repo_type="dataset")).json()
|
| 21 |
+
html = """"""
|
| 22 |
+
for row in batch(list(subdomains.keys()), n=4):
|
| 23 |
+
html += ("\n".join(["<div>"] +
|
| 24 |
+
[f"\t<iframe src='{subdomains[s]}' height='300', width='300'></iframe>" for s in row] +
|
| 25 |
+
["</div>"])
|
| 26 |
+
)
|
| 27 |
+
|
| 28 |
+
return html
|
| 29 |
+
|
| 30 |
+
|
| 31 |
+
|
| 32 |
+
app = FastAPI()
|
| 33 |
+
|
| 34 |
+
@app.get("/", response_class=HTMLResponse)
|
| 35 |
+
def index():
|
| 36 |
+
return embed_theme_spaces()
|
| 37 |
+
|
| 38 |
+
scheduler = BackgroundScheduler()
|
| 39 |
+
scheduler.add_job(func=process_spaces, trigger="interval", seconds=360)
|
| 40 |
+
scheduler.start()
|
| 41 |
+
|
| 42 |
+
if __name__ == "__main__":
|
| 43 |
+
uvicorn.run(app, port=7860)
|
| 44 |
+
|
processing.py
ADDED
|
@@ -0,0 +1,77 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
from typing import List, Dict, TypedDict
|
| 4 |
+
|
| 5 |
+
import huggingface_hub
|
| 6 |
+
from huggingface_hub.hf_api import SpaceInfo
|
| 7 |
+
import requests
|
| 8 |
+
from concurrent.futures import ThreadPoolExecutor
|
| 9 |
+
import os
|
| 10 |
+
import time
|
| 11 |
+
import random
|
| 12 |
+
import json
|
| 13 |
+
import datetime
|
| 14 |
+
|
| 15 |
+
class SpaceSubdomain(TypedDict):
|
| 16 |
+
name: str
|
| 17 |
+
subdomain: str
|
| 18 |
+
|
| 19 |
+
|
| 20 |
+
repo = huggingface_hub.Repository(
|
| 21 |
+
local_dir="data",
|
| 22 |
+
repo_type="dataset",
|
| 23 |
+
clone_from="freddyaboulton/gradio-theme-subdomains",
|
| 24 |
+
token=os.getenv("HF_TOKEN")
|
| 25 |
+
)
|
| 26 |
+
repo.git_pull()
|
| 27 |
+
|
| 28 |
+
|
| 29 |
+
api = huggingface_hub.HfApi(token=os.getenv("HF_TOKEN"))
|
| 30 |
+
|
| 31 |
+
def get_theme_preview_spaces() -> List[SpaceInfo]:
|
| 32 |
+
return api.list_spaces(filter="gradio-theme")
|
| 33 |
+
|
| 34 |
+
def get_subdomain(space_name: SpaceInfo) -> SpaceSubdomain | None:
|
| 35 |
+
time.sleep(random.random())
|
| 36 |
+
print(f"processing {space_name.id}")
|
| 37 |
+
if not space_name.id:
|
| 38 |
+
print(f"no space_name for {space_name}")
|
| 39 |
+
return None
|
| 40 |
+
subdomain: str | None = (
|
| 41 |
+
requests.get(
|
| 42 |
+
f"https://huggingface.co/api/spaces/{space_name.id}/host",
|
| 43 |
+
headers={"Authorization": f"Bearer {os.getenv('HF_TOKEN')}"}
|
| 44 |
+
)
|
| 45 |
+
.json()
|
| 46 |
+
.get("host")
|
| 47 |
+
)
|
| 48 |
+
if not subdomain:
|
| 49 |
+
print(f"no subdomain for {space_name.id}")
|
| 50 |
+
return None
|
| 51 |
+
return {'name': space_name.id, 'subdomain': subdomain}
|
| 52 |
+
|
| 53 |
+
|
| 54 |
+
def get_all_subdomains(spaces: List[SpaceInfo], subdomains: Dict[str, str]) -> Dict[str, str]:
|
| 55 |
+
|
| 56 |
+
not_parsed_yet = [space for space in spaces if space.id not in subdomains]
|
| 57 |
+
|
| 58 |
+
with ThreadPoolExecutor(max_workers=10) as executor:
|
| 59 |
+
new_subdomains = executor.map(get_subdomain, not_parsed_yet)
|
| 60 |
+
|
| 61 |
+
new_subdomains = {s["name"]: s["subdomain"] for s in new_subdomains if s}
|
| 62 |
+
|
| 63 |
+
new_subdomains.update(subdomains)
|
| 64 |
+
|
| 65 |
+
return new_subdomains
|
| 66 |
+
|
| 67 |
+
|
| 68 |
+
def process_spaces():
|
| 69 |
+
theme_spaces = get_theme_preview_spaces()
|
| 70 |
+
subdomains: Dict[str, str] = json.load(open("data/subdomains.json"))
|
| 71 |
+
|
| 72 |
+
all_subdomains = get_all_subdomains(theme_spaces, subdomains)
|
| 73 |
+
|
| 74 |
+
json.dump(all_subdomains, open("data/subdomains.json", "w"))
|
| 75 |
+
repo.push_to_hub(blocking=False, commit_message=f"Updating data at {datetime.datetime.now()}")
|
| 76 |
+
|
| 77 |
+
|
requirements.txt
ADDED
|
@@ -0,0 +1 @@
|
|
|
|
|
|
|
| 1 |
+
apscheduler
|