pt 1 of migration to new data frame

app.py

@@ -83,7 +83,7 @@ model_market_share_area = create_stacked_area_chart(
 )
 
 world_map = create_world_map(
-    country_concentration_df, "time", "metric", "value"
+    filtered_df
 )
 
 slider = create_range_slider(
@@ -208,19 +208,23 @@ app.layout = dmc.MantineProvider(
                     ]),
                     dcc.Tab(label='Leaderboard', children=[
                         create_leaderboard(
-                            filtered_df, country_concentration_df, author_concentration_df, model_concentration_df
+                            filtered_df
                         )
                     ]),
                     dcc.Tab(label='Model Tree Map', children=[
                         dcc.Graph(figure=tree_map)
                     ]),
-                    dcc.Tab(label='Model Characteristics', children=[
-                        dcc.Graph(id='language-concentration-chart'),
+                    dcc.Tab(label='Model Characteristics',children=[
                         html.Div([
-                            dcc.Dropdown(['Language Concentration', 'Architecture', 'License', 'Method'], 'Language Concentration', id='dropdown'),
-                        ], style={'marginTop': 6}),
-                    ]),
-                    dcc.Tab(label='Model Relationships', children=[
+                            html.H3("Language Concentration", style={'textAlign': 'center', 'marginBottom': 10}),
+                            dcc.Graph(figure=language_concentration_area),
+                            html.H3("License Distribution", style={'textAlign': 'center', 'marginBottom': 10}),
+                            dcc.Graph(figure=license_concentration_area),
+                            html.H3("Method Trends", style={'textAlign': 'center', 'marginBottom': 10}),
+                            dcc.Graph(figure=download_method_cumsum_line),
+                            html.H3("Architecture Trends", style={'textAlign': 'center', 'marginBottom': 10}),
+                            dcc.Graph(figure=download_arch_cumsum_line),
+                        ], style={'marginBottom': 12}),
                     ]),
                 ])
             ],
@@ -305,23 +309,6 @@ def update_stacked_area(value):
         return updated_fig
     return model_market_share_area
 
-    
-# Model Characteristics Tab
-# On dropdown change, update graph
-@app.callback(
-    Output('language-concentration-chart', 'figure'),
-    [Input('dropdown', 'value')]
-)
-def update_graph(selected_metric):
-    if selected_metric == 'Language Concentration':
-        return language_concentration_area
-    elif selected_metric == 'License':
-        return license_concentration_area
-    elif selected_metric == 'Method':
-        return download_method_cumsum_line
-    elif selected_metric == 'Architecture':
-        return download_arch_cumsum_line
-
 # Run the app
 if __name__ == '__main__':
     app.run(debug=True)

graphs/leaderboard.py

@@ -2,7 +2,7 @@ import pandas as pd
 from dash import html, dcc
 import base64
 
-def create_leaderboard(filtered_df, country_df, developer_df, model_df, start_time=None, end_time=None, top_n=10):
+def create_leaderboard(filtered_df, start_time=None, top_n=10):
     country_icon_map = {
         "USA": "🇺🇸",
         "China": "🇨🇳",
@@ -19,7 +19,8 @@ def create_leaderboard(filtered_df, country_df, developer_df, model_df, start_ti
         "Unknown": "❓",
         "Finland": "🇫🇮",
         "Lebanon": "🇱🇧",
-        "HF": "../assets/icons/hugging-face.png",
+        "User": "👤",
+        "International/Online": "🌐",
     }
 
     company_icon_map = {
@@ -30,51 +31,27 @@ def create_leaderboard(filtered_df, country_df, developer_df, model_df, start_ti
         "openai": "../assets/icons/openai.png",
     }
 
-    # Ensure datetime
-    for df in [country_df, developer_df, model_df]:
-        df["time"] = pd.to_datetime(df["time"])
-
-    # change any value that does not equal "org" to "user"
-    filtered_df["org_or_user"] = filtered_df["org_or_user"].where(filtered_df["org_or_user"] == "org", "user")
-
-    # Merge country info for developers/models
-    developer_df = developer_df.merge(
-        filtered_df[["country", "author", "org_or_user", "model", "downloads", "estimated_parameters"]].drop_duplicates(subset=["author"]),
-        left_on="metric", right_on="author", how="left"
-    ).drop(columns=["metric"])
-
-    model_df = model_df.merge(
-        filtered_df[["country", "author", "downloads", "org_or_user", "model", "merged_modality", "estimated_parameters"]].drop_duplicates(subset=["model"]),
-        left_on="metric", right_on="model", how="left"
-    ).drop(columns=["metric"])
-
-    # Rename metric columns
-    # country_df = country_df.rename(columns={"metric": "country"})
-    country_df = country_df.merge(
-        filtered_df[["country", "downloads", "estimated_parameters"]].drop_duplicates(subset=["country"]),
-        left_on="metric", right_on="country", how="left"
-    ).drop(columns=["metric"])
+    meta_cols_map = {
+        "org_country_single": ["org_country_single"],
+        "author": ["org_country_single", "author", "merged_country_groups_single"],
+        "model": ["org_country_single", "author", "merged_country_groups_single", "merged_modality", "downloads"]
+    }
 
     # Filter by time
-    start_time = start_time or country_df["time"].min()
-    end_time = end_time or country_df["time"].max()
-    country_df = country_df[(country_df["time"] >= start_time) & (country_df["time"] <= end_time)]
-    developer_df = developer_df[(developer_df["time"] >= start_time) & (developer_df["time"] <= end_time)]
-    model_df = model_df[(model_df["time"] >= start_time) & (model_df["time"] <= end_time)]
+    if start_time is not None:
+        filtered_df = filtered_df[(filtered_df["created"] >= start_time) & (filtered_df["time"] >= start_time)]
 
-    if country_df.empty and developer_df.empty and model_df.empty:
+    if filtered_df.empty:
         return html.Div("No data in selected range")
+    
+    # Merge HF and USA
+    filtered_df["org_country_single"] = filtered_df["org_country_single"].replace({"HF": "United States of America"})
+    # Merge International and Online
+    filtered_df["org_country_single"] = filtered_df["org_country_single"].replace({"International": "International/Online", "Online": "International/Online"})
 
     # Function to get top N leaderboard
-    def get_top_n_leaderboard(df, group_col, top_n=10):
-        top = (
-            df.groupby(group_col)["value"]
-            .sum()
-            .sort_values(ascending=False)
-            .head(top_n)
-            .reset_index()
-            .rename(columns={group_col: "Name", "value": "Total Value"})
-        )
+    def get_top_n_leaderboard(group_col, top_n=10):
+        top = filtered_df.groupby(group_col)["downloads"].sum().nlargest(top_n).reset_index().rename(columns={group_col: "Name", "downloads": "Total Value"})
         total_value = top["Total Value"].sum()
         top["% of total"] = top["Total Value"] / total_value * 100 if total_value else 0
 
@@ -83,13 +60,15 @@ def create_leaderboard(filtered_df, country_df, developer_df, model_df, start_ti
         download_top["Total Value"] = download_top["Total Value"].astype(int)
         download_top["% of total"] = download_top["% of total"].round(2)
 
+        top["Name"].replace("User", "user", inplace=True)
+
         # All relevant metadata columns
-        meta_cols = ["country", "author", "downloads", "org_or_user", "merged_modality", "estimated_parameters"]
+        meta_cols = meta_cols_map.get(group_col, [])
         # Collect all metadata per top n for each category (country, author, model)
         meta_map = {}
         download_map = {}
         for name in top["Name"]:
-            name_data = df[df[group_col] == name]
+            name_data = filtered_df[filtered_df[group_col] == name]
             meta_map[name] = {}
             download_map[name] = {}
             for col in meta_cols:
@@ -103,23 +82,27 @@ def create_leaderboard(filtered_df, country_df, developer_df, model_df, start_ti
             meta = meta_map.get(nm, {})
             chips = []
             # Countries
-            for c in meta.get("country", []):
+            for c in meta.get("org_country_single", []):
                 if c == "United States of America":
                     c = "USA"
+                if c == "user":
+                    c = "User"
                 chips.append((country_icon_map.get(c, ""), c))
             # Author
             for a in meta.get("author", []):
                 icon = company_icon_map.get(a, "")
                 if icon == "":
-                    if meta.get("org_or_user", ["user"])[0] == "org":
+                    if meta.get("merged_country_groups_single", ["User"])[0] != "User":
                         icon = "🏢"
                     else:
                         icon = "👤"
                 chips.append((icon, a))
             # Downloads
-            for d in meta.get("downloads", []):
-                if pd.notna(d):  # Check if d is not NaN
-                    chips.append(("⬇️", f"{int(d):,}"))
+            # Sum downloads if multiple entries
+            total_downloads = sum(d for d in meta.get("downloads", []) if pd.notna(d))  # Check if d is not NaN
+            if total_downloads:
+                chips.append(("⬇️", f"{int(total_downloads):,}"))
+
             # Modality
             for m in meta.get("merged_modality", []):
                 chips.append(("", m))
@@ -163,9 +146,9 @@ def create_leaderboard(filtered_df, country_df, developer_df, model_df, start_ti
         return top[["Name", "Metadata", "% of total"]], download_top
 
     # Build leaderboards
-    top_countries, download_top_countries = get_top_n_leaderboard(country_df, "country", top_n)
-    top_developers, download_top_developers = get_top_n_leaderboard(developer_df, "author", top_n)
-    top_models, download_top_models = get_top_n_leaderboard(model_df, "model", top_n)
+    top_countries, download_top_countries = get_top_n_leaderboard("org_country_single", top_n)
+    top_developers, download_top_developers = get_top_n_leaderboard("author", top_n)
+    top_models, download_top_models = get_top_n_leaderboard("model", top_n)
 
     # Chip renderer
     def chip(text, bg_color="#F0F0F0"):

graphs/model_market_share.py

@@ -1,3 +1,5 @@
+import numpy as np
+import pandas as pd
 import plotly.graph_objects as go
 from plotly.subplots import make_subplots
 
@@ -162,10 +164,17 @@ def create_stacked_area_chart(
 
 
 def create_world_map(
-    df, time_col="time", metric_col="metric", value_col="value", top_n_labels=10, start_time=None, end_time=None
+    df, top_n_labels=20
 ):
-    # Get all unique times and sort them
-    times = sorted(df[time_col].unique())
+    # Create a filtered_df with only countries
+    df = df[df['org_country_single'] != 'HF']
+    df = df[df['org_country_single'] != 'Online']
+    df = df[df['org_country_single'] != 'International']
+    df = df[df['org_country_single'] != 'user']
+
+    # Filter out models created after 2024-01-01 and downloads after 2024-01-01
+    # df = df[df['created'] > '2024-01-01']
+    # df = df[df['time'] > '2024-01-01']
 
     # Country code mapping
     country_code_map = {
@@ -214,8 +223,10 @@ def create_world_map(
         "Turkey": "TUR",
     }
 
-    df["country_code"] = df[metric_col].map(country_code_map)
-    mapped_data = df.dropna(subset=["country_code"])
+    df["country_code"] = df["org_country_single"].map(country_code_map)
+    df = df.dropna(subset=["country_code"])
+
+    # Fix country plot
 
     fig = make_subplots(
         rows=1,
@@ -223,42 +234,26 @@ def create_world_map(
         specs=[[{"type": "geo"}]],
     )
 
-    # Function to aggregate data for time range
-    def aggregate_time_range(start_time, end_time):
-        range_data = mapped_data[
-            (mapped_data[time_col] >= start_time) & (mapped_data[time_col] <= end_time)
-        ]
-        # Average values across time range
-        agg_data = (
-            range_data.groupby([metric_col, "country_code"])[value_col]
-            .mean()
-            .reset_index()
-        )
-        agg_data["percentage"] = agg_data[value_col] * 100
-        return agg_data.sort_values("percentage", ascending=False)
-
-    # Initial data if start or end time are not set (full range)
-    if start_time is None:
-        start_time = times[0]
-    if end_time is None:
-        end_time = times[-1]
-    initial_data = aggregate_time_range(start_time, end_time)
-    # top_countries = initial_data.head(top_n_labels)
+    downloads_by_country = df.groupby('org_country_single')['downloads'].sum().reset_index()
+    
+    # Prepare top countries for annotation
+    total_downloads = float(downloads_by_country['downloads'].sum())
+    downloads_by_country['pct'] = (downloads_by_country['downloads'] / total_downloads * 100.0)
 
     # Create hover text
     hover_text = []
-    for _, row in initial_data.iterrows():
+    for _, row in downloads_by_country.iterrows():
         hover_text.append(
-            f"<b>{row[metric_col]}</b><br>"
-            f"Avg Downloads: {row['percentage']:.1f}% of total<br>"
-            f"Avg Value: {row[value_col]:.6f}"
+            f"<b>{row['org_country_single']}</b><br>"
+            f"Avg Downloads: {row['pct']:.1f}% of total<br>"
+            f"Avg Value: {row['downloads']:.6f}"
         )
 
     # Add choropleth to plot
     fig.add_trace(
         go.Choropleth(
-            locations=initial_data["country_code"],
-            z=initial_data["percentage"],
+            locations=downloads_by_country["country_code"],
+            z=np.log10(downloads_by_country["pct"]),
             text=hover_text,
             hovertemplate="%{text}<extra></extra>",
             colorscale=[