Skip to content

Counter

counter

Token counting with tiktoken.

CounterBackend

Bases: StrEnum

Backend used to produce a token count.

Source code in packages/axm-smelt/src/axm_smelt/core/counter.py
Python
27
28
29
30
31
class CounterBackend(StrEnum):
    """Backend used to produce a token count."""

    TIKTOKEN = _TIKTOKEN_VALUE
    FALLBACK = _FALLBACK_VALUE

count(text, model='o200k_base')

Return the token count for text.

Uses tiktoken with model encoding. Falls back to len(text) // 4 when tiktoken is unavailable.

Source code in packages/axm-smelt/src/axm_smelt/core/counter.py
Python
56
57
58
59
60
61
62
63
def count(text: str, model: str = "o200k_base") -> int:
    """Return the token count for *text*.

    Uses tiktoken with *model* encoding.  Falls back to ``len(text) // 4``
    when tiktoken is unavailable.
    """
    n, _ = count_with_backend(text, model)
    return n

count_with_backend(text, model='o200k_base')

Return (token_count, backend) for text.

Falls back to len(text) // 4 when tiktoken is unavailable, and emits a single warning the first time the fallback path is taken in a process.

Source code in packages/axm-smelt/src/axm_smelt/core/counter.py
Python
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
def count_with_backend(
    text: str, model: str = "o200k_base"
) -> tuple[int, CounterBackend]:
    """Return ``(token_count, backend)`` for *text*.

    Falls back to ``len(text) // 4`` when tiktoken is unavailable, and emits
    a single warning the first time the fallback path is taken in a process.
    """
    global _warned
    try:
        enc = _ENC.get(model)
        if enc is None:
            enc = tiktoken.get_encoding(model)
            _ENC[model] = enc
        return len(enc.encode(text)), CounterBackend.TIKTOKEN
    except Exception:  # noqa: BLE001
        if not _warned:
            _log.warning("tiktoken unavailable, using approximate len//4 fallback")
            _warned = True
        return len(text) // 4, CounterBackend.FALLBACK

reset_warned()

Reset the one-shot warning flag (test seam).

Source code in packages/axm-smelt/src/axm_smelt/core/counter.py
Python
17
18
19
20
def reset_warned() -> None:
    """Reset the one-shot warning flag (test seam)."""
    global _warned
    _warned = False