Subword Tokenization Basics

# Merge the pair ("e", "s") -> "es", like one BPE step
tokens = ["l", "o", "w", "e", "s", "t"]
merged = []
i = 0
while i < len(tokens):
    if i < len(tokens) - 1 and tokens[i] == "e" and tokens[i+1] == "s":
        merged.append("es")
        i += 1
    else:
        merged.append(tokens[i])
        i += 1

print(merged)