Use JSONL-only experience storage with checkpoints every 300 games

README.md

@@ -25,7 +25,7 @@ After solving a word or using all six guesses, players submit a 0–100 rating b
 
 ## Dataset
 
-Rated records go to the private `QyrouNnet-AI/wordle-experiences` dataset, one new line per game in `experiences.jsonl`, only after the Hub commit succeeds. Individual JSON files are retained as archives. Concurrent submissions use a parent revision check and retries so earlier rows are preserved. Retries retain the first rating. No name, email, account identifier, or IP is included. Access logging is disabled; Hugging Face may independently maintain infrastructure logs.
+Rated records go to the private `QyrouNnet-AI/wordle-experiences` dataset, one new line per game in `experiences.jsonl`, only after the Hub commit succeeds. No per-game JSON files are created. At 300, 600, 900, and each further multiple of 300 experiences, an immutable cumulative JSONL snapshot is saved under `checkpoints/experiences-000300.jsonl`, `experiences-000600.jsonl`, and so on. The new row and checkpoint are saved together in one atomic Hub commit. The Dataset Viewer reads only the main table, so snapshots do not duplicate rows. Concurrent submissions use a parent revision check and retries so earlier rows are preserved. Retries retain the first rating. No name, email, account identifier, or IP is included. Access logging is disabled; Hugging Face may independently maintain infrastructure logs.
 
 Schema version 2 stores ground-truth word, accepted guess sequence, both selected letter hints (null for unavailable categories), revealed letters and their vowel/consonant type, reveal order, rating, outcome, ordered actions, and timings. `hint_mode` is `random_letters_v1`. Hint actions include `hint_type`, `variation`, `letter`, and `after_guess`.
 

backend/storage.py

@@ -8,6 +8,11 @@ from huggingface_hub.errors import HfHubHTTPError
 
 _lock = threading.Lock()
 TABLE = 'experiences.jsonl'
+CHECKPOINT_INTERVAL = 300
+
+
+def checkpoint_path(count):
+    return f'checkpoints/experiences-{count:06d}.jsonl' if count and count % CHECKPOINT_INTERVAL == 0 else None
 
 
 def rows_from(data):
@@ -35,21 +40,24 @@ def save_record(record):
             rating = existing_rating(rows, record['game_id'])
             if rating is not None:
                 return rating
-            path = directory / (record['game_id'] + '.json')
-            saved = json.loads(path.read_text()) if path.exists() else record
-            rows.append(saved)
+            rows.append(record)
+            data = encoded_rows(rows)
+            checkpoint = checkpoint_path(len(rows))
+            if checkpoint and not (directory / checkpoint).exists():
+                target = directory / checkpoint
+                target.parent.mkdir(parents=True, exist_ok=True)
+                temp_checkpoint = target.with_suffix('.tmp')
+                temp_checkpoint.write_bytes(data)
+                temp_checkpoint.replace(target)
             temporary = directory / (TABLE + '.tmp')
-            temporary.write_bytes(encoded_rows(rows))
+            temporary.write_bytes(data)
             temporary.replace(table)
-            if not path.exists():
-                path.write_text(json.dumps(saved, indent=2))
-            return saved['experience_rating']
+            return record['experience_rating']
         repo = os.environ.get('DATASET_REPO')
         token = os.environ.get('DATASET_TOKEN')
         if not repo or not token:
             raise RuntimeError('Feedback storage is not configured.')
         api = HfApi(token=token)
-        path = f"experiences/{record['game_id']}.json"
         for _ in range(8):
             info = api.dataset_info(repo)
             if not info.private:
@@ -62,16 +70,17 @@ def save_record(record):
             rating = existing_rating(rows, record['game_id'])
             if rating is not None:
                 return rating
-            saved = json.loads(read(path)) if path in files else record
-            rows.append(saved)
-            operations = [CommitOperationAdd(path_in_repo=TABLE, path_or_fileobj=encoded_rows(rows))]
-            if path not in files:
-                operations.append(CommitOperationAdd(path_in_repo=path, path_or_fileobj=json.dumps(saved, ensure_ascii=False).encode()))
+            rows.append(record)
+            data = encoded_rows(rows)
+            operations = [CommitOperationAdd(path_in_repo=TABLE, path_or_fileobj=data)]
+            checkpoint = checkpoint_path(len(rows))
+            if checkpoint and checkpoint not in files:
+                operations.append(CommitOperationAdd(path_in_repo=checkpoint, path_or_fileobj=data))
             try:
                 # Compare-and-swap protects rows written by other workers/replicas.
                 api.create_commit(repo, repo_type='dataset', operations=operations,
                                   parent_commit=revision, commit_message='Append a rated Wordle experience')
-                return saved['experience_rating']
+                return record['experience_rating']
             except HfHubHTTPError as exc:
                 if exc.response is None or exc.response.status_code not in (409, 412):
                     raise

scripts/migrate_experiences.py

@@ -1,46 +0,0 @@
-"""Build the JSONL table from current records, preserving existing rows and files."""
-import json
-from pathlib import Path
-from huggingface_hub import HfApi, CommitOperationAdd, hf_hub_download
-from huggingface_hub.errors import HfHubHTTPError
-
-
-def migrate(repo='QyrouNnet-AI/wordle-experiences'):
-    api = HfApi()
-    for _ in range(8):
-        info = api.dataset_info(repo)
-        if not info.private:
-            raise RuntimeError('Dataset must stay private.')
-        files = {item.rfilename for item in info.siblings}
-        def read(name):
-            return Path(hf_hub_download(repo, name, repo_type='dataset', revision=info.sha)).read_text()
-        rows = [json.loads(line) for line in read('experiences.jsonl').splitlines() if line.strip()] if 'experiences.jsonl' in files else []
-        ids = {row['game_id'] for row in rows}
-        for name in sorted(files):
-            if name.startswith('experiences/') and name.endswith('.json'):
-                row = json.loads(read(name))
-                if row['game_id'] not in ids:
-                    rows.append(row)
-                    ids.add(row['game_id'])
-        card = read('README.md')
-        if 'configs:' not in card:
-            card = card.replace('pretty_name: Wordle Experiences\n', 'pretty_name: Wordle Experiences\nconfigs:\n  - config_name: default\n    data_files:\n      - split: train\n        path: experiences.jsonl\n')
-        card = card.replace('as one JSON file in `experiences/`.', 'as one new line per game in `experiences.jsonl`. Individual JSON files in `experiences/` are retained as archives.')
-        if 'Concurrent submissions' not in card:
-            card += '\nConcurrent submissions use an atomic Hub commit with a parent revision check and retries. This preserves earlier rows. Retrying the same game ID does not add a duplicate or replace its original rating.\n'
-        table = ''.join(json.dumps(row, ensure_ascii=False, separators=(',', ':')) + '\n' for row in rows).encode()
-        try:
-            api.create_commit(repo, repo_type='dataset', parent_commit=info.sha,
-                              operations=[CommitOperationAdd(path_in_repo='experiences.jsonl', path_or_fileobj=table),
-                                          CommitOperationAdd(path_in_repo='README.md', path_or_fileobj=card.encode())],
-                              commit_message='Preserve experience records in an append-only JSONL table')
-            print(f'JSONL table contains {len(rows)} games; archive files preserved.')
-            return
-        except HfHubHTTPError as exc:
-            if exc.response is None or exc.response.status_code not in (409, 412):
-                raise
-    raise RuntimeError('Concurrent changes prevented migration; retry later.')
-
-
-if __name__ == '__main__':
-    migrate()

tests/test_game.py

@@ -114,9 +114,10 @@ def test_save_is_idempotent_and_preserves_first_rating(current, tmp_path, monkey
     record = game.experience_record(ended, 90)
     assert save_record(record) == 90
     assert save_record({**record, 'experience_rating': 1}) == 90
-    saved = json.loads((tmp_path / (record['game_id'] + '.json')).read_text())
+    saved = json.loads((tmp_path / 'experiences.jsonl').read_text().splitlines()[0])
     assert saved['experience_rating'] == 90
-    assert len(list(tmp_path.glob('*.json'))) == 1
+    assert not list(tmp_path.glob('*.json'))
+    assert len((tmp_path / 'experiences.jsonl').read_text().splitlines()) == 1
 
 def test_api_rating_failure_keeps_game(current, monkeypatch, tmp_path):
     import app as web
@@ -157,7 +158,7 @@ def test_cpu_new_game_and_hint_api_persist_letter_schema(monkeypatch, tmp_path):
         assert current['revealed'][0]['letter'] in 'ae'
         current = client.post('/api/guess', json={'token': current['token'], 'guess': 'crane'}).json()
         current = client.post('/api/rate', json={'token': current['token'], 'rating': 75}).json()
-        saved = json.loads((tmp_path / (current['id'] + '.json')).read_text())
+        saved = json.loads((tmp_path / 'experiences.jsonl').read_text().splitlines()[0])
         assert saved['schema_version'] == 2
         assert saved['hint_reveal_sequence'] == ['vowel']
         assert saved['actions'][0]['letter'] == saved['hints']['vowel']

tests/test_storage.py

@@ -54,3 +54,63 @@ def test_remote_conflict_reloads_other_players_row_before_appending(monkeypatch,
     assert [call['parent_commit'] for call in hub.calls] == ['v1', 'v2']
     assert storage.save_record(submitted) == 60
     assert len(hub.calls) == 2
+
+
+def test_local_checkpoints_are_cumulative_jsonl_and_immutable(monkeypatch, tmp_path):
+    monkeypatch.setenv('LOCAL_DATA_DIR', str(tmp_path))
+    rows = [{'game_id': str(i), 'experience_rating': 80} for i in range(1, 300)]
+    table = tmp_path / storage.TABLE
+    table.write_bytes(storage.encoded_rows(rows))
+    storage.save_record({'game_id': '300', 'experience_rating': 80})
+    checkpoint300 = tmp_path / 'checkpoints/experiences-000300.jsonl'
+    snapshot = checkpoint300.read_bytes()
+    assert snapshot == table.read_bytes()
+    assert len(storage.rows_from(snapshot)) == 300
+    storage.save_record({'game_id': '301', 'experience_rating': 80})
+    assert checkpoint300.read_bytes() == snapshot
+    assert len(list((tmp_path / 'checkpoints').iterdir())) == 1
+    storage.save_record({'game_id': '300', 'experience_rating': 1})
+    assert checkpoint300.read_bytes() == snapshot
+    rows = storage.rows_from(table.read_bytes())
+    rows.extend({'game_id': str(i), 'experience_rating': 80} for i in range(302, 600))
+    table.write_bytes(storage.encoded_rows(rows))
+    storage.save_record({'game_id': '600', 'experience_rating': 80})
+    checkpoint600 = tmp_path / 'checkpoints/experiences-000600.jsonl'
+    assert checkpoint600.read_bytes() == table.read_bytes()
+    assert len(storage.rows_from(checkpoint600.read_bytes())) == 600
+    assert checkpoint300.read_bytes() == snapshot
+    assert not list(tmp_path.rglob('*.json'))
+
+
+@pytest.mark.parametrize('initial_count', [0, 299, 300, 599])
+def test_hub_checkpoints_commit_atomically_with_new_line(monkeypatch, tmp_path, initial_count):
+    monkeypatch.delenv('LOCAL_DATA_DIR', raising=False)
+    monkeypatch.setenv('DATASET_REPO', 'test/private')
+    monkeypatch.setenv('DATASET_TOKEN', 'test-token')
+    rows = [{'game_id': str(i), 'experience_rating': 80} for i in range(initial_count)]
+    class FakeHub:
+        calls = []
+        def dataset_info(self, repo):
+            return SimpleNamespace(private=True, sha='snapshot',
+                                   siblings=[SimpleNamespace(rfilename=storage.TABLE)])
+        def create_commit(self, repo, **kwargs):
+            self.calls.append(kwargs)
+    hub = FakeHub()
+    monkeypatch.setattr(storage, 'HfApi', lambda **kwargs: hub)
+    table = tmp_path / 'remote.jsonl'
+    table.write_bytes(storage.encoded_rows(rows))
+    monkeypatch.setattr(storage, 'hf_hub_download', lambda *args, **kwargs: str(table))
+    storage.save_record({'game_id': 'new', 'experience_rating': 75})
+    assert len(hub.calls) == 1
+    commit = hub.calls[0]
+    assert commit['parent_commit'] == 'snapshot'
+    operations = commit['operations']
+    expected_paths = [storage.TABLE]
+    checkpoint = storage.checkpoint_path(initial_count + 1)
+    if checkpoint:
+        expected_paths.append(checkpoint)
+    assert [op.path_in_repo for op in operations] == expected_paths
+    assert all(op.path_in_repo.endswith('.jsonl') for op in operations)
+    assert len(storage.rows_from(operations[0].path_or_fileobj)) == initial_count + 1
+    if checkpoint:
+        assert operations[0].path_or_fileobj == operations[1].path_or_fileobj