from commonlid import Evaluator, get_model, get_dataset
Evaluator(
    models=[get_model("GlotLID")],
    datasets=[get_dataset("udhr")],
    output_dir="./results",
).run()

commonlid run --model GlotLID --dataset udhr --output-dir ./results

pip install commonlid                      # core deps + classical LID models
pip install "commonlid[llm]"               # + DSPy-based LLM evaluation
pip install "commonlid[afrolid]"           # + torch/transformers for AfroLID
pip install "commonlid[commonlingua]"      # + torch for the CommonLingua byte-level model
pip install "commonlid[notebooks]"         # + jupyterlab + matplotlib for paper_tables.ipynb
pip install "commonlid[all]"               # everything runtime-facing

uv add commonlid                           # runtime only
uv add "commonlid[all]"                    # all runtime extras

git clone https://github.com/commoncrawl/commonlid-eval.git
cd commonlid-eval
make install                               # uv sync --extra dev
make check                                 # ruff + mypy + pytest (matches CI)
source .venv/bin/activate

commonlid list-models
commonlid list-datasets

commonlid run \
  --model GlotLID --model OpenLID-v2 --model cld2 \
  --dataset commonlid --dataset flores_dev --dataset udhr \
  --output-dir ./results \
  --limit 500                  # cap samples per dataset (0 = all)

commonlid predict --model GlotLID --text "Le chat dort sur le canapé."
# {"text": "Le chat dort sur le canapé.", "pred": "fra", "model": "GlotLID"}

commonlid predict --model cld2 --text-file my_sentences.txt

commonlid export-csv --results-dir ./results --out ./results.csv

commonlid generate-support-matrix --out ./results/support_matrix.csv

commonlid run \
  --model GlotLID --model cld2 \
  --model dspy:azure/gpt-4o-mini \
  --api-base https://your-azure-endpoint.openai.azure.com/ \
  --api-version 2024-12-01-preview \
  --azure-ad-token \
  --temperature 0.7 \
  --dataset commonlid --dataset udhr \
  --output-dir ./results

from commonlid import Evaluator, get_model, get_dataset
results = Evaluator(
    models=[get_model("GlotLID"), get_model("cld2"), get_model("pyfranc")],
    datasets=[get_dataset("udhr"), get_dataset("commonlid")],
    output_dir="./results",
    batch_size=64,
    limit=500,          # optional cap per dataset
    use_cache=True,     # text-hash-keyed per-(model, dataset) cache
).run()
for r in results:
    s = r.summary()
    print(
        f"{r.model_id:12} on {r.dataset_id:10}  "
        f"macro F1={s['macro']['f1_gold_only']:.3f}  "
        f"samples/sec={s['samples_per_second']:.0f}"
    )

from commonlid import get_model
model = get_model("cld2")
preds = model.predict([
    "The quick brown fox jumps over the lazy dog",
    "Der schnelle braune Fuchs springt über den faulen Hund",
    "素早い茶色の狐が怠け者の犬を飛び越える",
])
assert preds == ["eng", "deu", "jpn"]

from commonlid import list_models, list_datasets
assert list_models() == [
    "AfroLID", "GlotLID", "OpenLID-v2", "cld2", "cld3",
    "commonlingua", "fasttext", "funlangid", "pyfranc",
]
assert list_datasets() == [
    "bibles_300", "bibles_300_nano",
    "commonlid", "commonlid_nano",
    "flores_dev", "flores_dev_nano",
    "smolsent_300", "smolsent_300_nano",
    "social_media_300", "social_media_300_nano",
    "udhr", "udhr_nano",
]

import math
from commonlid.metrics import (
    compute_per_language_metrics,
    macro_average,
    false_positive_rate,
)
ytrue = ["eng", "eng", "deu", "fra", "fra"]
ypred = ["eng", "eng", "deu", "fra", "spa"]
per_lang = compute_per_language_metrics(ytrue, ypred)
assert per_lang["fra"].precision == 1.0
assert per_lang["fra"].recall == 0.5
assert math.isclose(per_lang["fra"].f1, 2 / 3)
macro = macro_average(per_lang)
# Both views are returned. "gold-only" averages over languages with
# gt_count > 0 (the paper definition); "observed" averages over every
# language seen in either gold or predictions. Here ``spa`` is a
# spurious prediction (no gold) so the views diverge.
assert macro["n_languages_gold"] == 3      # eng, deu, fra
assert macro["n_languages_observed"] == 4  # + spa
assert macro["precision_gold_only"] == 1.0  # all gold langs have perfect precision
assert macro["precision_observed"] == 0.75  # spa drags the mean down
# 1 of 5 non-Spanish samples was mislabelled as Spanish
assert false_positive_rate(ytrue, ypred, language="spa") == 0.2

from commonlid import Evaluator, get_dataset
from commonlid.models.dspy_llm import DSPyLLMModel
model = DSPyLLMModel(
    llm_model_name="azure/gpt-4o-mini",
    api_base="https://your-endpoint.openai.azure.com/",
    api_version="2024-12-01-preview",
    azure_ad_token=True,     # uses DefaultAzureCredential
    temperature=0.7,
    batch_size=100,
    n_threads=4,
    cache_dir="./results/.dspy_cache",
)
Evaluator(
    models=[model],
    datasets=[get_dataset("commonlid")],
    output_dir="./results",
).run()

import json
from pathlib import Path
from commonlid.evaluation.results import load_summary
results_dir = Path("./results")
for summary_path in sorted(results_dir.rglob("summary.json")):
    s = load_summary(summary_path)
    print(s["model_id"], s["dataset_id"], s["macro"]["f1_gold_only"])
# Stream every per-sample prediction for one run:
preds_path = next(results_dir.rglob("predictions.jsonl"))
for line in preds_path.read_text().splitlines():
    row = json.loads(line)
    assert "gold" in row and "pred" in row and "correct" in row

# src/commonlid/models/my_model.py
from collections.abc import Sequence
from commonlid.core.lid_model import LIDModel
from commonlid.core.registry import get_model, register_model
@register_model
class MyModel(LIDModel):
    model_id = "my_model"
    def _predict_batch(self, texts: Sequence[str]) -> list[str | None]:
        # Return one ISO 639-3 code (or None for undetermined) per input.
        # `texts` arrives post-OpenLID-normer cleaning by default;
        # set `requires_preprocessing = False` to receive raw text.
        return ["eng"] * len(texts)
assert get_model("my_model").predict(["hi"]) == ["eng"]

from commonlid.models import my_model as _my_model  # noqa: F401

# src/commonlid/datasets/my_task.py
from commonlid.core.lid_dataset import LIDDataset
from commonlid.core.registry import get_dataset, register_dataset
@register_dataset
class MyTask(LIDDataset):
    dataset_id = "my_task"
    source_hf_repo = "me/my-lid-dataset"
    source_hf_revision = "abcdef1234567890..."   # pin a full git SHA
    source_hf_split = "test"
    text_column = "text"
    target_column = "iso639_3"
assert get_dataset("my_task").dataset_id == "my_task"

from commonlid.datasets import my_task as _my_task  # noqa: F401

{
  "schema_version": 1,
  "model_id": "GlotLID",
  "dataset_id": "udhr",
  "dataset_revision": "6908db2a27c296158da7e69782d15df911652184",
  "commonlid_version": "0.1.0",
  "python_version": "3.13.12",
  "platform": "macOS-15.2-arm64-arm-64bit",
  "timestamp": "2026-04-20T10:00:00+00:00",
  "limit": null,
  "n_samples": 2800,
  "n_samples_with_gold": 2800,
  "samples_per_second": 1842.3,
  "macro": {
    "f1_gold_only": 0.905, "precision_gold_only": 0.91, "recall_gold_only": 0.90,
    "n_languages_gold": 197,
    "f1_observed": 0.85, "precision_observed": 0.85, "recall_observed": 0.86,
    "n_languages_observed": 213
  },
  "micro": {
    "f1_gold_only": 0.88, "precision_gold_only": 0.88, "recall_gold_only": 0.88,
    "n_correct_gold": 2464, "n_predictions_gold": 2800, "n_gold_samples": 2800,
    "f1_observed": 0.86, "precision_observed": 0.84, "recall_observed": 0.88,
    "n_correct_observed": 2464, "n_predictions_observed": 2920
  },
  "per_language": {
    "eng": {
      "gt_count": 14, "predictions": 14, "correct": 14,
      "precision": 1.0, "recall": 1.0, "f1": 1.0
    }
  },
  "extra": {}
}

{"idx": 0, "text_hash": "abcd1234efgh5678", "gold": "eng", "pred": "eng", "correct": true}

make notebooks      # installs the [notebooks] extra and launches jupyter lab

make leaderboard                                # serve from ./data/results
# or against the live results dataset:
uv run commonlid leaderboard serve

hf auth login                                   # token with write access to the results dataset
make leaderboard-upload                         # opens a Pull Request from ./data/results
# Override the target with: make leaderboard-upload LEADERBOARD_REPO=other/repo LEADERBOARD_DIR=./elsewhere
# Optional: pass --skip-predictions via `uv run commonlid leaderboard upload ...` directly.

# One-time: create the Space (skip if already created via the web UI)
hf repo create --type space --space-sdk gradio commoncrawl/commonlid
# Push (or update) the entrypoint files
hf upload --repo-type=space commoncrawl/commonlid ./hf-space .

@misc{ortizsuarez2026commonlid,
  title = {CommonLID: Re-evaluating State-of-the-Art Language Identification Performance on Web Data},
  author = {Ortiz Suarez, Pedro and Burchell, Laurie and Arnett, Catherine and Mosquera-G{\'o}mez, Rafael and Hincapie-Monsalve, Sara and Vaughan, Thom and Stewart, Damian and Ostendorff, Malte and Abdulmumin, Idris and Marivate, Vukosi and Muhammad, Shamsuddeen Hassan and Tonja, Atnafu Lambebo and Al-Khalifa, Hend and Ghezaiel Hammouda, Nadia and Otiende, Verrah and Wong, Tack Hwa and Saydaliev, Jakhongir and Nobakhtian, Melika and Habibi, Muhammad Ravi Shulthan and Kranti, Chalamalasetti and Muchemi, Carol and Nguyen, Khang and Adam, Faisal Muhammad and Salim, Luis Frentzen and Alqifari, Reem and Amol, Cynthia and Imperial, Joseph Marvin and Kesen, Ilker and Mustafid, Ahmad and Stepachev, Pavel and Choshen, Leshem and Anugraha, David and Nayel, Hamada and Yimam, Seid Muhie and Putra, Vallerie Alexandra and Nguyen, My Chiffon and Wasi, Azmine Toushik and Vadithya, Gouthami and van der Goot, Rob and ar C'horr, Lanwenn and Dua, Karan and Yates, Andrew and Bangera, Mithil and Bangera, Yeshil and Patel, Hitesh Laxmichand and Okabe, Shu and Ilasariya, Fenal Ashokbhai and Gaynullin, Dmitry and Winata, Genta Indra and Li, Yiyuan and Mart{\'\i}nez, Juan Pablo and Agarwal, Amit and Hanif, Ikhlasul Akmal and Abu Ahmad, Raia and Adenuga, Esther and Tjiaranata, Filbert Aurelian and Buaphet, Weerayut and Anugraha, Michael and Vajjala, Sowmya and Rice, Benjamin and Amirudin, Azril Hafizi and Alabi, Jesujoba O. and Panda, Srikant and Toughrai, Yassine and Kyomuhendo, Bruhan and Ruffinelli, Daniel and A, Akshata and Goul{\~a}o, Manuel and Zhou, Ej and Franco Ramirez, Ingrid Gabriela and Aggazzotti, Cristina and Dobler, Konstantin and Kevin, Jun and Pag{\`e}s, Quentin and Andrews, Nicholas and Ibrahim, Nuhu and Ruckdeschel, Mattes and Keleg, Amr and Zhang, Mike and Muziri, Casper and Samuel, Saron and Takeshita, Sotaro and Kerdthaisong, Kun and Foppiano, Luca and Dent, Rasul and Green, Tommaso and Wali, Ahmad Mustapha and Makaaka, Kamohelo and Feliren, Vicky and Idris, Inshirah and Celikkanat, Hande and Abubakar, Abdulhamid and Maillard, Jean and Sagot, Beno{\^i}t and Cl{\'e}rice, Thibault and Murray, Kenton and Luger, Sarah},
  year = {2026},
  eprint = {2601.18026},
  archivePrefix = {arXiv},
  primaryClass = {cs.CL},
  doi = {10.48550/arXiv.2601.18026},
  url = {https://arxiv.org/abs/2601.18026},
}