Don't commitschedule when locally + use local database for testing
Browse files
app.py
CHANGED
|
@@ -6,7 +6,7 @@ import gradio as gr
|
|
| 6 |
import numpy as np
|
| 7 |
from PIL import Image
|
| 8 |
import random
|
| 9 |
-
from db import compute_elo_scores, get_all_votes, add_vote
|
| 10 |
import json
|
| 11 |
from pathlib import Path
|
| 12 |
from uuid import uuid4
|
|
@@ -16,6 +16,8 @@ import time
|
|
| 16 |
from datasets import load_dataset
|
| 17 |
from huggingface_hub import CommitScheduler
|
| 18 |
|
|
|
|
|
|
|
| 19 |
token = os.getenv("HUGGINGFACE_HUB_TOKEN")
|
| 20 |
|
| 21 |
# Load datasets
|
|
@@ -31,15 +33,16 @@ load_dotenv()
|
|
| 31 |
JSON_DATASET_DIR = Path("data/json_dataset")
|
| 32 |
JSON_DATASET_DIR.mkdir(parents=True, exist_ok=True)
|
| 33 |
|
| 34 |
-
# Initialize CommitScheduler for Hugging Face
|
| 35 |
-
scheduler =
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
-
|
| 40 |
-
|
| 41 |
-
|
| 42 |
-
|
|
|
|
| 43 |
|
| 44 |
def fetch_elo_scores():
|
| 45 |
"""Fetch and log Elo scores."""
|
|
@@ -216,8 +219,10 @@ def gradio_interface():
|
|
| 216 |
try:
|
| 217 |
logging.debug("Adding vote data to the database: %s", vote_data)
|
| 218 |
|
| 219 |
-
|
| 220 |
-
|
|
|
|
|
|
|
| 221 |
except Exception as e:
|
| 222 |
logging.error("Error recording vote: %s", str(e))
|
| 223 |
|
|
@@ -270,6 +275,10 @@ def gradio_interface():
|
|
| 270 |
|
| 271 |
def dump_database_to_json():
|
| 272 |
"""Dump the database to a JSON file and upload it to Hugging Face."""
|
|
|
|
|
|
|
|
|
|
|
|
|
| 273 |
votes = get_all_votes()
|
| 274 |
json_data = [
|
| 275 |
{
|
|
@@ -301,10 +310,13 @@ def schedule_dump_database(interval=60):
|
|
| 301 |
logging.info("Database dump completed. Sleeping for %d seconds.", interval)
|
| 302 |
time.sleep(interval)
|
| 303 |
|
| 304 |
-
|
| 305 |
-
|
| 306 |
-
|
| 307 |
-
|
|
|
|
|
|
|
|
|
|
| 308 |
|
| 309 |
if __name__ == "__main__":
|
| 310 |
schedule_dump_database() # Start the periodic database dump
|
|
|
|
| 6 |
import numpy as np
|
| 7 |
from PIL import Image
|
| 8 |
import random
|
| 9 |
+
from db import compute_elo_scores, get_all_votes, add_vote, is_running_in_space
|
| 10 |
import json
|
| 11 |
from pathlib import Path
|
| 12 |
from uuid import uuid4
|
|
|
|
| 16 |
from datasets import load_dataset
|
| 17 |
from huggingface_hub import CommitScheduler
|
| 18 |
|
| 19 |
+
|
| 20 |
+
|
| 21 |
token = os.getenv("HUGGINGFACE_HUB_TOKEN")
|
| 22 |
|
| 23 |
# Load datasets
|
|
|
|
| 33 |
JSON_DATASET_DIR = Path("data/json_dataset")
|
| 34 |
JSON_DATASET_DIR.mkdir(parents=True, exist_ok=True)
|
| 35 |
|
| 36 |
+
# Initialize CommitScheduler for Hugging Face only if running in space
|
| 37 |
+
scheduler = None
|
| 38 |
+
if is_running_in_space():
|
| 39 |
+
scheduler = CommitScheduler(
|
| 40 |
+
repo_id="bgsys/votes_datasets_test2",
|
| 41 |
+
repo_type="dataset",
|
| 42 |
+
folder_path=JSON_DATASET_DIR,
|
| 43 |
+
path_in_repo="data",
|
| 44 |
+
token=token
|
| 45 |
+
)
|
| 46 |
|
| 47 |
def fetch_elo_scores():
|
| 48 |
"""Fetch and log Elo scores."""
|
|
|
|
| 219 |
try:
|
| 220 |
logging.debug("Adding vote data to the database: %s", vote_data)
|
| 221 |
|
| 222 |
+
# Only add vote if running in space
|
| 223 |
+
if is_running_in_space():
|
| 224 |
+
result = add_vote(vote_data)
|
| 225 |
+
logging.info("Vote successfully recorded in the database with ID: %s", result["id"])
|
| 226 |
except Exception as e:
|
| 227 |
logging.error("Error recording vote: %s", str(e))
|
| 228 |
|
|
|
|
| 275 |
|
| 276 |
def dump_database_to_json():
|
| 277 |
"""Dump the database to a JSON file and upload it to Hugging Face."""
|
| 278 |
+
if not is_running_in_space():
|
| 279 |
+
logging.info("Not running in Hugging Face Spaces. Skipping database dump.")
|
| 280 |
+
return
|
| 281 |
+
|
| 282 |
votes = get_all_votes()
|
| 283 |
json_data = [
|
| 284 |
{
|
|
|
|
| 310 |
logging.info("Database dump completed. Sleeping for %d seconds.", interval)
|
| 311 |
time.sleep(interval)
|
| 312 |
|
| 313 |
+
if is_running_in_space():
|
| 314 |
+
logging.info("Initializing database dump scheduler with interval: %d seconds.", interval)
|
| 315 |
+
thread = threading.Thread(target=run, daemon=True)
|
| 316 |
+
thread.start()
|
| 317 |
+
logging.info("Database dump scheduler started.")
|
| 318 |
+
else:
|
| 319 |
+
logging.info("Not running in Hugging Face Spaces. Database dump scheduler not started.")
|
| 320 |
|
| 321 |
if __name__ == "__main__":
|
| 322 |
schedule_dump_database() # Start the periodic database dump
|
db.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
|
|
| 1 |
from sqlalchemy import create_engine, Column, Integer, String, DateTime
|
| 2 |
from sqlalchemy.ext.declarative import declarative_base
|
| 3 |
from sqlalchemy.orm import sessionmaker, Session
|
|
@@ -6,8 +7,14 @@ import pandas as pd
|
|
| 6 |
import uuid
|
| 7 |
from rating_systems import compute_elo
|
| 8 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 9 |
|
| 10 |
-
DATABASE_URL = "sqlite:///./data/newvotes.db" # Example with SQLite, replace with PostgreSQL for production
|
| 11 |
engine = create_engine(DATABASE_URL, connect_args={"check_same_thread": False})
|
| 12 |
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
| 13 |
Base = declarative_base()
|
|
|
|
| 1 |
+
import os
|
| 2 |
from sqlalchemy import create_engine, Column, Integer, String, DateTime
|
| 3 |
from sqlalchemy.ext.declarative import declarative_base
|
| 4 |
from sqlalchemy.orm import sessionmaker, Session
|
|
|
|
| 7 |
import uuid
|
| 8 |
from rating_systems import compute_elo
|
| 9 |
|
| 10 |
+
def is_running_in_space():
|
| 11 |
+
return "SPACE_ID" in os.environ
|
| 12 |
+
|
| 13 |
+
if is_running_in_space():
|
| 14 |
+
DATABASE_URL = "sqlite:///./data/newvotes.db"
|
| 15 |
+
else:
|
| 16 |
+
DATABASE_URL = "sqlite:///./data/local.db"
|
| 17 |
|
|
|
|
| 18 |
engine = create_engine(DATABASE_URL, connect_args={"check_same_thread": False})
|
| 19 |
SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine)
|
| 20 |
Base = declarative_base()
|