tokenizers

💥 Fast State-of-the-Art Tokenizers optimized for Research and Production

10,947 stars Rust 1 file · ~2,895 tokens #bert #gpt #language-model #natural-language-processing #natural-language-understanding #nlp #transformers

tokenizers MCP Server

MCP server for huggingface/tokenizers

Ready-to-Use Client Configurations

{
  "mcpServers": {
    "tokenizers": {
      "command": "npx",
      "args": [
        "-y",
        "@huggingface/tokenizers"
      ]
    }
  }
}

{
  "mcpServers": {
    "tokenizers": {
      "command": "npx",
      "args": [
        "-y",
        "@huggingface/tokenizers"
      ]
    }
  }
}

{
  "mcpServers": {
    "tokenizers": {
      "command": "npx",
      "args": [
        "-y",
        "@huggingface/tokenizers"
      ]
    }
  }
}

{
  "mcpServers": {
    "tokenizers": {
      "command": "npx",
      "args": [
        "-y",
        "@huggingface/tokenizers"
      ],
      "disabled": false,
      "autoApprove": []
    }
  }
}

{
  "mcpServers": {
    "tokenizers": {
      "command": "npx",
      "args": [
        "-y",
        "@huggingface/tokenizers"
      ]
    }
  }
}

docker run -i --rm mcp/tokenizers