initial shift to duckdb

app.py

@@ -1,6 +1,8 @@
 from dash import Dash, html, dcc, Input, Output, State
 import pandas as pd
 import dash_mantine_components as dmc
+import duckdb
+import time
 from graphs.leaderboard import (
     create_leaderboard,
     get_top_n_leaderboard,
@@ -11,12 +13,44 @@ from graphs.leaderboard import (
 app = Dash()
 server = app.server
 
-# Load pre-processed data frames
-filtered_df = pd.read_pickle("data_frames/filtered_df.pkl")
+# DuckDB connection (global)
+con = duckdb.connect(database=':memory:', read_only=False)
+
+# Load parquet file from Hugging Face using DuckDB
+HF_DATASET_ID = "emsesc/open_model_evolution_data"
+hf_parquet_url = "https://huggingface.co/datasets/emsesc/open_model_evolution_data/resolve/main/filtered_df.parquet"
+
+print(f"Attempting to connect to dataset from Hugging Face Hub: {HF_DATASET_ID}")
+try:
+    overall_start_time = time.time()
+    
+    # Install and load httpfs extension for remote file access
+    con.execute("INSTALL httpfs;")
+    con.execute("LOAD httpfs;")
+    
+    # Create a view that references the remote parquet file
+    con.execute(f"""
+        CREATE OR REPLACE VIEW filtered_df AS 
+        SELECT * FROM read_parquet('{hf_parquet_url}')
+    """)
+    
+    # Get column list and basic info
+    columns = con.execute("DESCRIBE filtered_df").fetchdf()
+    print("Columns:", columns['column_name'].tolist())
+    
+    # Get time range for slider
+    time_range = con.execute("SELECT MIN(time) as min_time, MAX(time) as max_time FROM filtered_df").fetchdf()
+    start_dt = pd.to_datetime(time_range['min_time'].iloc[0])
+    end_dt = pd.to_datetime(time_range['max_time'].iloc[0])
+    
+    msg = f"Successfully connected to dataset in {time.time() - overall_start_time:.2f}s."
+    print(msg)
+except Exception as e:
+    err_msg = f"Failed to load dataset. Error: {e}"
+    print(err_msg)
+    raise
 
 # Create a dcc slider for time range selection by year (readable marks)
-start_dt = filtered_df["time"].min()
-end_dt = filtered_df["time"].max()
 start_ts = int(start_dt.timestamp())
 end_ts = int(end_dt.timestamp())
 
@@ -48,10 +82,6 @@ time_slider = dmc.RangeSlider(
     marks=marks,
     style={"width": "70%", "margin": "0 auto"},
     labelAlwaysOn=False,
-    # thumbChildren=[
-    #     dmc.Text(id="time-slider-thumb-from-label", size="xs", children="Hello"),
-    #     dmc.Text(id="time-slider-thumb-to-label", size="xs"),
-    # ]
 )
 
 # App layout
@@ -167,7 +197,7 @@ app.layout = dmc.MantineProvider(
                 # Intro / description below header (kept but styled to match layout)
                 # Title
                 html.Div(
-                    children="Model Leaderboard",  # Change this to your desired title
+                    children="Model Leaderboard",
                     style={
                         "fontSize": 40,
                         "fontWeight": "700",
@@ -180,7 +210,7 @@ app.layout = dmc.MantineProvider(
                 html.Div(
                     children=[
                         html.Button(
-                            "Read the paper",  # Change this to your desired button text
+                            "Read the paper",
                             id="my-button",
                             style={
                                 "padding": "10px 20px",
@@ -269,7 +299,6 @@ app.layout = dmc.MantineProvider(
                         "gap": "24px",
                         "padding": "32px",
                         "alignItems": "flex-start",
-                        # 'margin': '24px auto 64px',     # centered horizontally
                         "marginLeft": "100px",
                         "marginRight": "100px",
                         "backgroundColor": "#FFFBF9",
@@ -281,7 +310,7 @@ app.layout = dmc.MantineProvider(
                         dcc.Tabs(
                             id="leaderboard-tabs",
                             value="Countries",
-                            children=[  # wrap Tabs here
+                            children=[
                                 dcc.Tab(
                                     label="Countries",
                                     value="Countries",
@@ -297,10 +326,10 @@ app.layout = dmc.MantineProvider(
                                         "border": "none",
                                         "padding": "10px 18px",
                                         "fontWeight": "700",
-                                        "borderBottom": "3px solid #082030",  # underline only
+                                        "borderBottom": "3px solid #082030",
                                     },
                                     children=[
-                                        create_leaderboard(filtered_df, "countries")
+                                        create_leaderboard(con, "countries")
                                     ],
                                 ),
                                 dcc.Tab(
@@ -321,7 +350,7 @@ app.layout = dmc.MantineProvider(
                                         "borderBottom": "3px solid #082030",
                                     },
                                     children=[
-                                        create_leaderboard(filtered_df, "developers")
+                                        create_leaderboard(con, "developers")
                                     ],
                                 ),
                                 dcc.Tab(
@@ -342,7 +371,7 @@ app.layout = dmc.MantineProvider(
                                         "borderBottom": "3px solid #082030",
                                     },
                                     children=[
-                                        create_leaderboard(filtered_df, "models")
+                                        create_leaderboard(con, "models")
                                     ],
                                 ),
                             ],
@@ -355,7 +384,6 @@ app.layout = dmc.MantineProvider(
                         "marginBottom": "64px",
                         "marginLeft": "50px",
                         "marginRight": "50px",
-                        # 'maxWidth': '1250px',
                     },
                 ),
             ],
@@ -370,12 +398,62 @@ app.layout = dmc.MantineProvider(
 
 # Callbacks for interactivity
 # -- helper utilities to consolidate duplicated callback logic --
-def _apply_time_slider(slider_value):
+def _get_filtered_top_n_from_duckdb(slider_value, group_col, top_n):
+    """
+    Query DuckDB directly to get top N entries with metadata
+    This minimizes data transfer by doing aggregation in DuckDB
+    """
+    # Build time filter clause
+    time_filter = ""
     if slider_value and len(slider_value) == 2:
         start = pd.to_datetime(slider_value[0], unit="s")
         end = pd.to_datetime(slider_value[1], unit="s")
-        return filtered_df[(filtered_df["time"] >= start) & (filtered_df["time"] <= end)]
-    return filtered_df
+        time_filter = f"WHERE time >= '{start}' AND time <= '{end}'"
+    
+    # Apply country replacements in the query
+    country_case = """
+        CASE 
+            WHEN org_country_single = 'HF' THEN 'United States of America'
+            WHEN org_country_single = 'International' THEN 'International/Online'
+            WHEN org_country_single = 'Online' THEN 'International/Online'
+            ELSE org_country_single
+        END as org_country_single
+    """
+    
+    # Build the aggregation query to get top N with all needed metadata
+    # This query groups by the target column and aggregates downloads
+    # while collecting all metadata we need for chips
+    query = f"""
+    WITH base_data AS (
+        SELECT 
+            {group_col},
+            {country_case},
+            author,
+            merged_country_groups_single,
+            merged_modality,
+            downloads,
+            estimated_parameters,
+            model
+        FROM filtered_df
+        {time_filter}
+    ),
+    aggregated AS (
+        SELECT 
+            {group_col} as name,
+            SUM(downloads) as total_downloads
+        FROM base_data
+        GROUP BY {group_col}
+        ORDER BY total_downloads DESC
+        LIMIT {top_n}
+    )
+    SELECT 
+        b.*
+    FROM base_data b
+    INNER JOIN aggregated a ON b.{group_col} = a.name
+    ORDER BY a.total_downloads DESC
+    """
+    
+    return con.execute(query).fetchdf()
 
 def _leaderboard_callback_logic(n_clicks, slider_value, current_label, group_col, filename, default_label="▼ Show Top 50", chip_color="#F0F9FF"):
     # Normalize label on first load
@@ -393,14 +471,14 @@ def _leaderboard_callback_logic(n_clicks, slider_value, current_label, group_col
     else:
         top_n, new_label = 10, "▼ Show Top 50"
 
-    # Apply time filter and build table
-    df_time = _apply_time_slider(slider_value)
-    df, download_df = get_top_n_leaderboard(df_time, group_col, top_n)
+    # Get filtered and aggregated data directly from DuckDB
+    df_filtered = _get_filtered_top_n_from_duckdb(slider_value, group_col, top_n)
+    
+    # Process the already-filtered data
+    df, download_df = get_top_n_leaderboard(df_filtered, group_col, top_n)
     return render_table_content(df, download_df, chip_color=chip_color, filename=filename), new_label
 # -- end helpers --
 
-# ...existing code...
-
 # Callbacks for interactivity (modularized)
 @app.callback(
     Output("top_countries-table", "children"),
@@ -468,4 +546,4 @@ def update_range_labels(values):
 
 # Run the app
 if __name__ == "__main__":
-    app.run(debug=True)
+    app.run(debug=True)

graphs/leaderboard.py

@@ -47,6 +47,33 @@ country_icon_map = {
     "Switzerland": "🇨🇭",
     "User": "👤",
     "International/Online": "🌐",
+    "Spain": "🇪🇸",
+    "Sweden": "🇸🇪",
+    "Norway": "🇳🇴",
+    "Denmark": "🇩🇰",
+    "Austria": "🇦🇹",
+    "Belgium": "🇧🇪",
+    "Poland": "🇵🇱",
+    "Turkey": "🇹🇷",
+    "Mexico": "🇲🇽",
+    "Argentina": "🇦🇷",
+    "Thailand": "🇹🇭",
+    "Indonesia": "🇮🇩",
+    "Malaysia": "🇲🇾",
+    "Philippines": "🇵🇭",
+    "Egypt": "🇪🇬",
+    "South Africa": "🇿🇦",
+    "New Zealand": "🇳🇿",
+    "Ireland": "🇮🇪",
+    "Portugal": "🇵🇹",
+    "Greece": "🇬🇷",
+    "Czech Republic": "🇨🇿",
+    "Romania": "🇷🇴",
+    "Ukraine": "🇺🇦",
+    "United Arab Emirates": "🇦🇪",
+    "Saudi Arabia": "🇸🇦",
+    "Pakistan": "🇵🇰",
+    "Bangladesh": "🇧🇩",
 }
 
 company_icon_map = {
@@ -370,8 +397,20 @@ def render_table(
     )
 
 
-# Function to get top N leaderboard
+# Function to get top N leaderboard (now accepts pandas DataFrame from DuckDB query)
 def get_top_n_leaderboard(filtered_df, group_col, top_n=10):
+    """
+    Get top N entries for a leaderboard
+    
+    Args:
+        filtered_df: Pandas DataFrame (already filtered by time from DuckDB query)
+        group_col: Column to group by
+        top_n: Number of top entries to return
+        
+    Returns:
+        tuple: (display_df, download_df)
+    """
+    # Group by and get top N
     top = (
         filtered_df.groupby(group_col)["downloads"]
         .sum()
@@ -379,6 +418,7 @@ def get_top_n_leaderboard(filtered_df, group_col, top_n=10):
         .reset_index()
         .rename(columns={group_col: "Name", "downloads": "Total Value"})
     )
+    
     total_value = top["Total Value"].sum()
     top["% of total"] = top["Total Value"] / total_value * 100 if total_value else 0
 
@@ -387,17 +427,21 @@ def get_top_n_leaderboard(filtered_df, group_col, top_n=10):
     download_top["Total Value"] = download_top["Total Value"].astype(int)
     download_top["% of total"] = download_top["% of total"].round(2)
 
-    top["Name"].replace("User", "user")
+    # Replace "User" in names
+    top["Name"] = top["Name"].replace("User", "user")
 
     # All relevant metadata columns
     meta_cols = meta_cols_map.get(group_col, [])
+    
     # Collect all metadata per top n for each category (country, author, model)
     meta_map = {}
     download_map = {}
+    
     for name in top["Name"]:
         name_data = filtered_df[filtered_df[group_col] == name]
         meta_map[name] = {}
         download_map[name] = {}
+        
         for col in meta_cols:
             if col in name_data.columns:
                 unique_vals = name_data[col].unique()
@@ -408,13 +452,15 @@ def get_top_n_leaderboard(filtered_df, group_col, top_n=10):
     def build_metadata(nm):
         meta = meta_map.get(nm, {})
         chips = []
+        
         # Countries
         for c in meta.get("org_country_single", []):
             if c == "United States of America":
                 c = "USA"
             if c == "user":
                 c = "User"
-            chips.append((country_icon_map.get(c, ""), c))
+            chips.append((country_icon_map.get(c, "🌍"), c))
+        
         # Author
         for a in meta.get("author", []):
             icon = company_icon_map.get(a, "")
@@ -424,21 +470,22 @@ def get_top_n_leaderboard(filtered_df, group_col, top_n=10):
                 else:
                     icon = "👤"
             chips.append((icon, a))
+        
         # Downloads
-        # Sum downloads if multiple entries
         total_downloads = sum(
             d for d in meta.get("downloads", []) if pd.notna(d)
-        )  # Check if d is not NaN
+        )
         if total_downloads:
             chips.append(("⬇️", f"{int(total_downloads):,}"))
 
         # Modality
         for m in meta.get("merged_modality", []):
-            chips.append(("", m))
+            if pd.notna(m):
+                chips.append(("", m))
 
         # Estimated Parameters
         for p in meta.get("estimated_parameters", []):
-            if pd.notna(p):  # Check if p is not NaN
+            if pd.notna(p):
                 if p >= 1e9:
                     p_str = f"{p / 1e9:.1f}B"
                 elif p >= 1e6:
@@ -446,28 +493,32 @@ def get_top_n_leaderboard(filtered_df, group_col, top_n=10):
                 elif p >= 1e3:
                     p_str = f"{p / 1e3:.1f}K"
                 else:
-                    p_str = str(p)
+                    p_str = str(int(p))
                 chips.append(("⚙️", p_str))
+        
         return chips
 
-    # Function to create downloadable dataframe
+    # Function to create downloadable dataframe metadata
     def build_download_metadata(nm):
         meta = download_map.get(nm, {})
         download_info = {}
+        
         for col in meta_cols:
-            # don't add empty columns
             if col not in meta or not meta[col]:
                 continue
+            
             vals = meta.get(col, [])
             if vals:
-                # Join list into a single string for CSV
-                download_info[col] = ", ".join(str(v) for v in vals)
+                download_info[col] = ", ".join(str(v) for v in vals if pd.notna(v))
             else:
                 download_info[col] = ""
+        
         return download_info
 
     # Apply metadata builder to top dataframe
     top["Metadata"] = top["Name"].astype(object).apply(build_metadata)
+    
+    # Build download dataframe with metadata
     download_info_list = [build_download_metadata(nm) for nm in download_top["Name"]]
     download_info_df = pd.DataFrame(download_info_list)
     download_top = pd.concat([download_top, download_info_df], axis=1)
@@ -475,52 +526,123 @@ def get_top_n_leaderboard(filtered_df, group_col, top_n=10):
     return top[["Name", "Metadata", "% of total"]], download_top
 
 
-def create_leaderboard(filtered_df, board_type, top_n=10):
-    if filtered_df.empty:
-        return html.Div("No data in selected range")
-
-    # Merge HF and USA
-    filtered_df["org_country_single"] = filtered_df["org_country_single"].replace(
-        {"HF": "United States of America"}
-    )
-    # Merge International and Online
-    filtered_df["org_country_single"] = filtered_df["org_country_single"].replace(
-        {"International": "International/Online", "Online": "International/Online"}
-    )
-
-    # Build leaderboards
-    top_countries, download_top_countries = get_top_n_leaderboard(
-        filtered_df, "org_country_single", top_n
+def get_top_n_from_duckdb(con, group_col, top_n=10, time_filter=None):
+    """
+    Query DuckDB directly to get top N entries with minimal data transfer
+    
+    Args:
+        con: DuckDB connection object
+        group_col: Column to group by
+        top_n: Number of top entries
+        time_filter: Optional tuple of (start_timestamp, end_timestamp)
+        
+    Returns:
+        Pandas DataFrame with only the rows needed for top N
+    """
+    # Build time filter clause
+    time_clause = ""
+    if time_filter:
+        start = pd.to_datetime(time_filter[0], unit="s")
+        end = pd.to_datetime(time_filter[1], unit="s")
+        time_clause = f"WHERE time >= '{start}' AND time <= '{end}'"
+    
+    # Apply country replacements in the query
+    country_case = """
+        CASE 
+            WHEN org_country_single = 'HF' THEN 'United States of America'
+            WHEN org_country_single = 'International' THEN 'International/Online'
+            WHEN org_country_single = 'Online' THEN 'International/Online'
+            ELSE org_country_single
+        END as org_country_single
+    """
+    
+    # Optimized query: first find top N, then get only those rows
+    query = f"""
+    WITH base_data AS (
+        SELECT 
+            {group_col},
+            {country_case},
+            author,
+            merged_country_groups_single,
+            merged_modality,
+            downloads,
+            estimated_parameters,
+            model
+        FROM filtered_df
+        {time_clause}
+    ),
+    top_items AS (
+        SELECT 
+            {group_col} as name,
+            SUM(downloads) as total_downloads
+        FROM base_data
+        GROUP BY {group_col}
+        ORDER BY total_downloads DESC
+        LIMIT {top_n}
     )
-    top_developers, download_top_developers = get_top_n_leaderboard(
-        filtered_df, "author", top_n
-    )
-    top_models, download_top_models = get_top_n_leaderboard(filtered_df, "model", top_n)
-
-    if board_type == "countries":
-        return render_table(
-            top_countries,
-            download_top_countries,
-            "Top Countries",
-            chip_color="#F0F9FF",
-            bar_color="#082030",
-            filename="top_countries",
-        )
-    elif board_type == "developers":
-        return render_table(
-            top_developers,
-            download_top_developers,
-            "Top Developers",
-            chip_color="#F0F9FF",
-            bar_color="#082030",
-            filename="top_developers",
-        )
-    else:
-        return render_table(
-            top_models,
-            download_top_models,
-            "Top Models",
-            chip_color="#F0F9FF",
-            bar_color="#082030",
-            filename="top_models",
-        )
+    SELECT 
+        b.*
+    FROM base_data b
+    INNER JOIN top_items t ON b.{group_col} = t.name
+    ORDER BY t.total_downloads DESC
+    """
+    
+    try:
+        return con.execute(query).fetchdf()
+    except Exception as e:
+        print(f"Error querying DuckDB: {e}")
+        return pd.DataFrame()
+
+
+def create_leaderboard(con, board_type, top_n=10):
+    """
+    Create leaderboard using DuckDB connection with optimized queries
+    
+    Args:
+        con: DuckDB connection object
+        board_type: Type of leaderboard ('countries', 'developers', 'models')
+        top_n: Number of top entries to display
+        
+    Returns:
+        Dash HTML component with the leaderboard table
+    """
+    # Map board type to column name
+    column_map = {
+        "countries": "org_country_single",
+        "developers": "author",
+        "models": "model"
+    }
+    
+    title_map = {
+        "countries": "Top Countries",
+        "developers": "Top Developers",
+        "models": "Top Models"
+    }
+    
+    filename_map = {
+        "countries": "top_countries",
+        "developers": "top_developers",
+        "models": "top_models"
+    }
+    
+    group_col = column_map.get(board_type)
+    if not group_col:
+        return html.Div(f"Unknown board type: {board_type}")
+    
+    # Get only the top N rows from DuckDB
+    filtered_df = get_top_n_from_duckdb(con, group_col, top_n)
+    
+    if filtered_df.empty:
+        return html.Div("No data available")
+    
+    # Process the already-filtered data
+    top_data, download_data = get_top_n_leaderboard(filtered_df, group_col, top_n)
+    
+    return render_table(
+        top_data,
+        download_data,
+        title_map[board_type],
+        chip_color="#F0F9FF",
+        bar_color="#082030",
+        filename=filename_map[board_type],
+    )

requirements.txt

@@ -3,4 +3,6 @@ dash
 plotly
 gunicorn
 dash-mantine-components
-dash-bootstrap-components
+dash-bootstrap-components
+pyarrow
+duckdb